mirror of
https://github.com/gocsaf/csaf.git
synced 2025-12-22 11:55:40 +01:00
Add CSAF downloader
* Dense and refactor ROLIE code in aggregator a bit. * Move advisory file processor to csaf package. * Fix minor typo on main readme
This commit is contained in:
Sascha L. Teichmann
GitHub
parent
640ef64df9
commit
b359fd0a62
20 changed files with 929 additions and 239 deletions
484
cmd/csaf_downloader/downloader.go
Normal file
484
cmd/csaf_downloader/downloader.go
Normal file
|
|
@ -0,0 +1,484 @@
|
|||
// This file is Free Software under the MIT License
|
||||
// without warranty, see README.md and LICENSES/MIT.txt for details.
|
||||
//
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
|
||||
// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v2/crypto"
|
||||
"github.com/csaf-poc/csaf_distribution/csaf"
|
||||
"github.com/csaf-poc/csaf_distribution/util"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type downloader struct {
|
||||
client util.Client
|
||||
opts *options
|
||||
directory string
|
||||
keys []*crypto.KeyRing
|
||||
}
|
||||
|
||||
func (d *downloader) httpClient() util.Client {
|
||||
|
||||
if d.client != nil {
|
||||
return d.client
|
||||
}
|
||||
|
||||
hClient := http.Client{}
|
||||
|
||||
var tlsConfig tls.Config
|
||||
if d.opts.Insecure {
|
||||
tlsConfig.InsecureSkipVerify = true
|
||||
hClient.Transport = &http.Transport{
|
||||
TLSClientConfig: &tlsConfig,
|
||||
}
|
||||
}
|
||||
|
||||
var client util.Client
|
||||
|
||||
if d.opts.Verbose {
|
||||
client = &util.LoggingClient{Client: &hClient}
|
||||
} else {
|
||||
client = &hClient
|
||||
}
|
||||
|
||||
if d.opts.Rate == nil {
|
||||
d.client = client
|
||||
return client
|
||||
}
|
||||
|
||||
d.client = &util.LimitingClient{
|
||||
Client: client,
|
||||
Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1),
|
||||
}
|
||||
|
||||
return d.client
|
||||
}
|
||||
|
||||
func (d *downloader) loadProviderMetadataDirectly(path string) *csaf.LoadedProviderMetadata {
|
||||
client := d.httpClient()
|
||||
resp, err := client.Get(path)
|
||||
if err != nil {
|
||||
log.Printf("Error fetching '%s': %v\n", path, err)
|
||||
return nil
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Printf(
|
||||
"Error fetching '%s': %s (%d)\n", path, resp.Status, resp.StatusCode)
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var doc interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil {
|
||||
log.Printf("Decoding '%s' as JSON failed: %v\n", path, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
errors, err := csaf.ValidateProviderMetadata(doc)
|
||||
if err != nil {
|
||||
log.Printf("Schema validation of '%s' failed: %v\n", path, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
log.Printf(
|
||||
"Schema validation of '%s' leads to %d issues.\n", path, len(errors))
|
||||
return nil
|
||||
}
|
||||
|
||||
return &csaf.LoadedProviderMetadata{
|
||||
Document: doc,
|
||||
URL: path,
|
||||
}
|
||||
}
|
||||
|
||||
func (d *downloader) download(domain string) error {
|
||||
|
||||
var lpmd *csaf.LoadedProviderMetadata
|
||||
|
||||
if strings.HasPrefix(domain, "https://") {
|
||||
lpmd = d.loadProviderMetadataDirectly(domain)
|
||||
} else {
|
||||
lpmd = csaf.LoadProviderMetadataForDomain(
|
||||
d.httpClient(), domain, func(format string, args ...interface{}) {
|
||||
log.Printf(
|
||||
"Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...)
|
||||
})
|
||||
}
|
||||
|
||||
if lpmd == nil {
|
||||
return fmt.Errorf("no provider-metadata.json found for '%s'", domain)
|
||||
}
|
||||
|
||||
base, err := url.Parse(lpmd.URL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid URL '%s': %v", lpmd.URL, err)
|
||||
}
|
||||
|
||||
eval := util.NewPathEval()
|
||||
|
||||
if err := d.loadOpenPGPKeys(
|
||||
d.httpClient(),
|
||||
eval,
|
||||
lpmd.Document,
|
||||
base,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
afp := csaf.NewAdvisoryFileProcessor(
|
||||
d.httpClient(),
|
||||
eval,
|
||||
lpmd.Document,
|
||||
base)
|
||||
|
||||
return afp.Process(d.downloadFiles)
|
||||
}
|
||||
|
||||
func (d *downloader) loadOpenPGPKeys(
|
||||
client util.Client,
|
||||
eval *util.PathEval,
|
||||
doc interface{},
|
||||
base *url.URL,
|
||||
) error {
|
||||
|
||||
src, err := eval.Eval("$.public_openpgp_keys", doc)
|
||||
if err != nil {
|
||||
// no keys.
|
||||
return nil
|
||||
}
|
||||
|
||||
var keys []csaf.PGPKey
|
||||
if err := util.ReMarshalJSON(&keys, src); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try to load
|
||||
|
||||
for i := range keys {
|
||||
key := &keys[i]
|
||||
if key.URL == nil {
|
||||
continue
|
||||
}
|
||||
up, err := url.Parse(*key.URL)
|
||||
if err != nil {
|
||||
log.Printf("Invalid URL '%s': %v", *key.URL, err)
|
||||
continue
|
||||
}
|
||||
|
||||
u := base.ResolveReference(up).String()
|
||||
|
||||
res, err := client.Get(u)
|
||||
if err != nil {
|
||||
log.Printf("Fetching public OpenPGP key %s failed: %v.", u, err)
|
||||
continue
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
log.Printf("Fetching public OpenPGP key %s status code: %d (%s)",
|
||||
u, res.StatusCode, res.Status)
|
||||
continue
|
||||
}
|
||||
|
||||
ckey, err := func() (*crypto.Key, error) {
|
||||
defer res.Body.Close()
|
||||
return crypto.NewKeyFromArmoredReader(res.Body)
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Reading public OpenPGP key %s failed: %v", u, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if !strings.EqualFold(ckey.GetFingerprint(), string(key.Fingerprint)) {
|
||||
log.Printf(
|
||||
"Fingerprint of public OpenPGP key %s does not match remotely loaded.", u)
|
||||
continue
|
||||
}
|
||||
keyring, err := crypto.NewKeyRing(ckey)
|
||||
if err != nil {
|
||||
log.Printf("Creating store for public OpenPGP key %s failed: %v.", u, err)
|
||||
continue
|
||||
}
|
||||
d.keys = append(d.keys, keyring)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *downloader) downloadFiles(label csaf.TLPLabel, files []csaf.AdvisoryFile) error {
|
||||
|
||||
client := d.httpClient()
|
||||
|
||||
var data bytes.Buffer
|
||||
|
||||
var lastDir string
|
||||
|
||||
for _, file := range files {
|
||||
|
||||
u, err := url.Parse(file.URL())
|
||||
if err != nil {
|
||||
log.Printf("Ignoring invalid URL: %s: %v\n", file.URL(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Ignore not confirming filenames.
|
||||
filename := filepath.Base(u.Path)
|
||||
if !util.ConfirmingFileName(filename) {
|
||||
log.Printf("Not confirming filename %q. Ignoring.\n", filename)
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := client.Get(file.URL())
|
||||
if err != nil {
|
||||
log.Printf("WARN: cannot get '%s': %v\n", file.URL(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Printf("WARN: cannot load %s: %s (%d)\n",
|
||||
file.URL(), resp.Status, resp.StatusCode)
|
||||
continue
|
||||
}
|
||||
|
||||
var (
|
||||
writers []io.Writer
|
||||
s256, s512 hash.Hash
|
||||
s256Data, s512Data []byte
|
||||
remoteSHA256, remoteSHA512 []byte
|
||||
signData []byte
|
||||
)
|
||||
|
||||
// Only hash when we have a remote counter part we can compare it with.
|
||||
if remoteSHA256, s256Data, err = d.loadHash(file.SHA256URL()); err != nil {
|
||||
if d.opts.Verbose {
|
||||
log.Printf("WARN: cannot fetch %s: %v\n", file.SHA256URL(), err)
|
||||
}
|
||||
} else {
|
||||
s256 = sha256.New()
|
||||
writers = append(writers, s256)
|
||||
}
|
||||
|
||||
if remoteSHA512, s512Data, err = d.loadHash(file.SHA512URL()); err != nil {
|
||||
if d.opts.Verbose {
|
||||
log.Printf("WARN: cannot fetch %s: %v\n", file.SHA512URL(), err)
|
||||
}
|
||||
} else {
|
||||
s512 = sha512.New()
|
||||
writers = append(writers, s512)
|
||||
}
|
||||
|
||||
// Remember the data as we need to store it to file later.
|
||||
data.Reset()
|
||||
writers = append(writers, &data)
|
||||
|
||||
// Download the advisory and hash it.
|
||||
hasher := io.MultiWriter(writers...)
|
||||
|
||||
var doc interface{}
|
||||
|
||||
if err := func() error {
|
||||
defer resp.Body.Close()
|
||||
tee := io.TeeReader(resp.Body, hasher)
|
||||
return json.NewDecoder(tee).Decode(&doc)
|
||||
}(); err != nil {
|
||||
log.Printf("Downloading %s failed: %v", file.URL(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Compare the checksums.
|
||||
if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) {
|
||||
log.Printf("SHA256 checksum of %s does not match.\n", file.URL())
|
||||
continue
|
||||
}
|
||||
|
||||
if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) {
|
||||
log.Printf("SHA512 checksum of %s does not match.\n", file.URL())
|
||||
continue
|
||||
}
|
||||
|
||||
// Only check signature if we have loaded keys.
|
||||
if len(d.keys) > 0 {
|
||||
var sign *crypto.PGPSignature
|
||||
sign, signData, err = d.loadSignature(file.SignURL())
|
||||
if err != nil {
|
||||
if d.opts.Verbose {
|
||||
log.Printf("downloading signature '%s' failed: %v\n",
|
||||
file.SignURL(), err)
|
||||
}
|
||||
}
|
||||
if sign != nil {
|
||||
if !d.checkSignature(data.Bytes(), sign) {
|
||||
log.Printf("Cannot verify signature for %s\n", file.URL())
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate against CSAF schema.
|
||||
errors, err := csaf.ValidateCSAF(doc)
|
||||
if err != nil {
|
||||
log.Printf("Failed to validate %s: %v", file.URL(), err)
|
||||
continue
|
||||
}
|
||||
if len(errors) > 0 {
|
||||
log.Printf("CSAF file %s has %d validation errors.", file.URL(), len(errors))
|
||||
continue
|
||||
}
|
||||
|
||||
// Write advisory to file
|
||||
|
||||
newDir := path.Join(d.directory, string(label))
|
||||
if newDir != lastDir {
|
||||
if err := os.MkdirAll(newDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
lastDir = newDir
|
||||
}
|
||||
|
||||
path := filepath.Join(lastDir, filename)
|
||||
if err := os.WriteFile(path, data.Bytes(), 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write hash sums.
|
||||
if s256Data != nil {
|
||||
if err := os.WriteFile(path+".sha256", s256Data, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s512Data != nil {
|
||||
if err := os.WriteFile(path+".sha512", s512Data, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Write signature.
|
||||
if signData != nil {
|
||||
if err := os.WriteFile(path+".asc", signData, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Written advisory '%s'.\n", path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *downloader) checkSignature(data []byte, sign *crypto.PGPSignature) bool {
|
||||
pm := crypto.NewPlainMessage(data)
|
||||
t := crypto.GetUnixTime()
|
||||
for _, key := range d.keys {
|
||||
if err := key.VerifyDetached(pm, sign, t); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *downloader) loadSignature(p string) (*crypto.PGPSignature, []byte, error) {
|
||||
resp, err := d.httpClient().Get(p)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, nil, fmt.Errorf(
|
||||
"fetching signature from '%s' failed: %s (%d)", p, resp.Status, resp.StatusCode)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sign, err := crypto.NewPGPSignatureFromArmored(string(data))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return sign, data, nil
|
||||
}
|
||||
|
||||
func (d *downloader) loadHash(p string) ([]byte, []byte, error) {
|
||||
resp, err := d.httpClient().Get(p)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, nil, fmt.Errorf(
|
||||
"fetching hash from '%s' failed: %s (%d)", p, resp.Status, resp.StatusCode)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var data bytes.Buffer
|
||||
tee := io.TeeReader(resp.Body, &data)
|
||||
hash, err := util.HashFromReader(tee)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return hash, data.Bytes(), nil
|
||||
}
|
||||
|
||||
// prepareDirectory ensures that the working directory
|
||||
// exists and is setup properly.
|
||||
func (d *downloader) prepareDirectory() error {
|
||||
// If no special given use current working directory.
|
||||
if d.opts.Directory == nil {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.directory = dir
|
||||
return nil
|
||||
}
|
||||
// Use given directory
|
||||
if _, err := os.Stat(*d.opts.Directory); err != nil {
|
||||
// If it does not exist create it.
|
||||
if os.IsNotExist(err) {
|
||||
if err = os.MkdirAll(*d.opts.Directory, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
d.directory = *d.opts.Directory
|
||||
return nil
|
||||
}
|
||||
|
||||
// run performs the downloads for all the given domains.
|
||||
func (d *downloader) run(domains []string) error {
|
||||
|
||||
if err := d.prepareDirectory(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, domain := range domains {
|
||||
if err := d.download(domain); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue