SystemSh0cker/gocsaf
1
0
Fork 0
mirror of https://github.com/gocsaf/csaf.git synced 2025-12-22 11:55:40 +01:00
Code Issues Releases Activity Actions

Refactor downloader to allow usage as library

Browse source 
- Allow to specify custom logger
- Add callback for downloaded documents
- Separate config of downloader CLI and library
- Remove `Forwarder` tests that test the handler
This commit is contained in:
koplas 2024-06-19 15:55:05 +02:00 • committed by koplas
parent fae4fdeabe
commit 513282a7a8
No known key found for this signature in database
6 changed files with 514 additions and 481 deletions
Download patch file Download diff file Expand all files Collapse all files

176
lib/downloader/downloader.go
View file

@ -22,10 +22,7 @@ import (
"log/slog"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
@ -48,6 +45,18 @@ type Downloader struct {
stats stats
}
// DownloadedDocument contains the document data with additional metadata.
type DownloadedDocument struct {
Data bytes.Buffer
S256Data []byte
S512Data []byte
SignData []byte
InitialReleaseDate time.Time
Filename string
ValStatus ValidationStatus
Label csaf.TLPLabel
}
// failedValidationDir is the name of the sub folder
// where advisories are stored that fail validation in
// unsafe mode.
@ -94,15 +103,17 @@ func (d *Downloader) addStats(o *stats) {
}
// logRedirect logs redirects of the http client.
func logRedirect(req *http.Request, via []*http.Request) error {
vs := make([]string, len(via))
for i, v := range via {
vs[i] = v.URL.String()
func logRedirect(logger *slog.Logger) func(req *http.Request, via []*http.Request) error {
return func(req *http.Request, via []*http.Request) error {
vs := make([]string, len(via))
for i, v := range via {
vs[i] = v.URL.String()
}
logger.Debug("Redirecting",
"to", req.URL.String(),
"via", strings.Join(vs, " -> "))
return nil
}
slog.Debug("Redirecting",
"to", req.URL.String(),
"via", strings.Join(vs, " -> "))
return nil
}
func (d *Downloader) httpClient() util.Client {
@ -110,7 +121,7 @@ func (d *Downloader) httpClient() util.Client {
hClient := http.Client{}
if d.cfg.verbose() {
hClient.CheckRedirect = logRedirect
hClient.CheckRedirect = logRedirect(d.cfg.Logger)
}
var tlsConfig tls.Config
@ -118,8 +129,8 @@ func (d *Downloader) httpClient() util.Client {
tlsConfig.InsecureSkipVerify = true
}
if len(d.cfg.clientCerts) != 0 {
tlsConfig.Certificates = d.cfg.clientCerts
if len(d.cfg.ClientCerts) != 0 {
tlsConfig.Certificates = d.cfg.ClientCerts
}
hClient.Transport = &http.Transport{
@ -140,7 +151,7 @@ func (d *Downloader) httpClient() util.Client {
if d.cfg.verbose() {
client = &util.LoggingClient{
Client: client,
Log: httpLog("downloader"),
Log: httpLog("downloader", d.cfg.Logger),
}
}
@ -156,9 +167,9 @@ func (d *Downloader) httpClient() util.Client {
}
// httpLog does structured logging in a [util.LoggingClient].
func httpLog(who string) func(string, string) {
func httpLog(who string, logger *slog.Logger) func(string, string) {
return func(method, url string) {
slog.Debug("http",
logger.Debug("http",
"who", who,
"method", method,
"url", url)
@ -176,7 +187,7 @@ func (d *Downloader) enumerate(domain string) error {
for _, pmd := range lpmd {
if d.cfg.verbose() {
for i := range pmd.Messages {
slog.Debug("Enumerating provider-metadata.json",
d.cfg.Logger.Debug("Enumerating provider-metadata.json",
"domain", domain,
"message", pmd.Messages[i].Message)
}
@ -188,7 +199,7 @@ func (d *Downloader) enumerate(domain string) error {
// print the results
doc, err := json.MarshalIndent(docs, "", " ")
if err != nil {
slog.Error("Couldn't marshal PMD document json")
d.cfg.Logger.Error("Couldn't marshal PMD document json")
}
fmt.Println(string(doc))
@ -211,7 +222,7 @@ func (d *Downloader) download(ctx context.Context, domain string) error {
return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain)
} else if d.cfg.verbose() {
for i := range lpmd.Messages {
slog.Debug("Loading provider-metadata.json",
d.cfg.Logger.Debug("Loading provider-metadata.json",
"domain", domain,
"message", lpmd.Messages[i].Message)
}
@ -241,7 +252,7 @@ func (d *Downloader) download(ctx context.Context, domain string) error {
// Do we need time range based filtering?
if d.cfg.Range != nil {
slog.Debug("Setting up filter to accept advisories within",
d.cfg.Logger.Debug("Setting up filter to accept advisories within",
"timerange", d.cfg.Range)
afp.AgeAccept = d.cfg.Range.Contains
}
@ -306,7 +317,6 @@ func (d *Downloader) loadOpenPGPKeys(
base *url.URL,
expr *util.PathEval,
) error {
src, err := expr.Eval("$.public_openpgp_keys", doc)
if err != nil {
// no keys.
@ -331,7 +341,7 @@ func (d *Downloader) loadOpenPGPKeys(
}
up, err := url.Parse(*key.URL)
if err != nil {
slog.Warn("Invalid URL",
d.cfg.Logger.Warn("Invalid URL",
"url", *key.URL,
"error", err)
continue
@ -341,14 +351,14 @@ func (d *Downloader) loadOpenPGPKeys(
res, err := client.Get(u)
if err != nil {
slog.Warn(
d.cfg.Logger.Warn(
"Fetching public OpenPGP key failed",
"url", u,
"error", err)
continue
}
if res.StatusCode != http.StatusOK {
slog.Warn(
d.cfg.Logger.Warn(
"Fetching public OpenPGP key failed",
"url", u,
"status_code", res.StatusCode,
@ -362,7 +372,7 @@ func (d *Downloader) loadOpenPGPKeys(
}()
if err != nil {
slog.Warn(
d.cfg.Logger.Warn(
"Reading public OpenPGP key failed",
"url", u,
"error", err)
@ -370,14 +380,14 @@ func (d *Downloader) loadOpenPGPKeys(
}
if !strings.EqualFold(ckey.GetFingerprint(), string(key.Fingerprint)) {
slog.Warn(
d.cfg.Logger.Warn(
"Fingerprint of public OpenPGP key does not match remotely loaded",
"url", u)
continue
}
if d.keys == nil {
if keyring, err := crypto.NewKeyRing(ckey); err != nil {
slog.Warn(
d.cfg.Logger.Warn(
"Creating store for public OpenPGP key failed",
"url", u,
"error", err)
@ -394,18 +404,18 @@ func (d *Downloader) loadOpenPGPKeys(
// logValidationIssues logs the issues reported by the advisory schema validation.
func (d *Downloader) logValidationIssues(url string, errors []string, err error) {
if err != nil {
slog.Error("Failed to validate",
d.cfg.Logger.Error("Failed to validate",
"url", url,
"error", err)
return
}
if len(errors) > 0 {
if d.cfg.verbose() {
slog.Error("CSAF file has validation errors",
d.cfg.Logger.Error("CSAF file has validation errors",
"url", url,
"error", strings.Join(errors, ", "))
} else {
slog.Error("CSAF file has validation errors",
d.cfg.Logger.Error("CSAF file has validation errors",
"url", url,
"count", len(errors))
}
@ -424,10 +434,8 @@ func (d *Downloader) downloadWorker(
var (
client = d.httpClient()
data bytes.Buffer
lastDir string
initialReleaseDate time.Time
dateExtract = util.TimeMatcher(&initialReleaseDate, time.RFC3339)
lower = strings.ToLower(string(label))
stats = stats{}
expr = util.NewPathEval()
)
@ -451,14 +459,14 @@ nextAdvisory:
u, err := url.Parse(file.URL())
if err != nil {
stats.downloadFailed++
slog.Warn("Ignoring invalid URL",
d.cfg.Logger.Warn("Ignoring invalid URL",
"url", file.URL(),
"error", err)
continue
}
if d.cfg.ignoreURL(file.URL()) {
slog.Debug("Ignoring URL", "url", file.URL())
d.cfg.Logger.Debug("Ignoring URL", "url", file.URL())
continue
}
@ -466,7 +474,7 @@ nextAdvisory:
filename := filepath.Base(u.Path)
if !util.ConformingFileName(filename) {
stats.filenameFailed++
slog.Warn("Ignoring none conforming filename",
d.cfg.Logger.Warn("Ignoring none conforming filename",
"filename", filename)
continue
}
@ -474,7 +482,7 @@ nextAdvisory:
resp, err := client.Get(file.URL())
if err != nil {
stats.downloadFailed++
slog.Warn("Cannot GET",
d.cfg.Logger.Warn("Cannot GET",
"url", file.URL(),
"error", err)
continue
@ -482,7 +490,7 @@ nextAdvisory:
if resp.StatusCode != http.StatusOK {
stats.downloadFailed++
slog.Warn("Cannot load",
d.cfg.Logger.Warn("Cannot load",
"url", file.URL(),
"status", resp.Status,
"status_code", resp.StatusCode)
@ -491,7 +499,7 @@ nextAdvisory:
// Warn if we do not get JSON.
if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
slog.Warn("Content type is not 'application/json'",
d.cfg.Logger.Warn("Content type is not 'application/json'",
"url", file.URL(),
"content_type", ct)
}
@ -506,7 +514,7 @@ nextAdvisory:
// Only hash when we have a remote counter part we can compare it with.
if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
slog.Warn("Cannot fetch SHA256",
d.cfg.Logger.Warn("Cannot fetch SHA256",
"url", file.SHA256URL(),
"error", err)
} else {
@ -515,7 +523,7 @@ nextAdvisory:
}
if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
slog.Warn("Cannot fetch SHA512",
d.cfg.Logger.Warn("Cannot fetch SHA512",
"url", file.SHA512URL(),
"error", err)
} else {
@ -538,7 +546,7 @@ nextAdvisory:
return json.NewDecoder(tee).Decode(&doc)
}(); err != nil {
stats.downloadFailed++
slog.Warn("Downloading failed",
d.cfg.Logger.Warn("Downloading failed",
"url", file.URL(),
"error", err)
continue
@ -570,7 +578,7 @@ nextAdvisory:
var sign *crypto.PGPSignature
sign, signData, err = loadSignature(client, file.SignURL())
if err != nil {
slog.Warn("Downloading signature failed",
d.cfg.Logger.Warn("Downloading signature failed",
"url", file.SignURL(),
"error", err)
}
@ -624,7 +632,7 @@ nextAdvisory:
}
// Run all the validations.
valStatus := notValidatedValidationStatus
valStatus := NotValidatedValidationStatus
for _, check := range []func() error{
s256Check,
s512Check,
@ -634,14 +642,14 @@ nextAdvisory:
remoteValidatorCheck,
} {
if err := check(); err != nil {
slog.Error("Validation check failed", "error", err)
valStatus.update(invalidValidationStatus)
d.cfg.Logger.Error("Validation check failed", "error", err)
valStatus.update(InvalidValidationStatus)
if d.cfg.ValidationMode == ValidationStrict {
continue nextAdvisory
}
}
}
valStatus.update(validValidationStatus)
valStatus.update(ValidValidationStatus)
// Send to Forwarder
if d.Forwarder != nil {
@ -651,15 +659,6 @@ nextAdvisory:
string(s256Data),
string(s512Data))
}
if d.cfg.NoStore {
// Do not write locally.
if valStatus == validValidationStatus {
stats.succeeded++
}
continue
}
if err := expr.Extract(
`$.document.tracking.initial_release_date`, dateExtract, false, doc,
); err != nil {
@ -669,61 +668,26 @@ nextAdvisory:
}
initialReleaseDate = initialReleaseDate.UTC()
// Advisories that failed validation are stored in a special folder.
var newDir string
if valStatus != validValidationStatus {
newDir = path.Join(d.cfg.Directory, failedValidationDir)
download := DownloadedDocument{
Data: data,
S256Data: s256Data,
S512Data: s512Data,
SignData: signData,
InitialReleaseDate: initialReleaseDate,
Filename: filename,
ValStatus: valStatus,
Label: label,
}
err = d.cfg.DownloadHandler(download)
if err != nil {
errorCh <- err
} else {
newDir = d.cfg.Directory
stats.succeeded++
}
// Do we have a configured destination folder?
if d.cfg.Folder != "" {
newDir = path.Join(newDir, d.cfg.Folder)
} else {
newDir = path.Join(newDir, lower, strconv.Itoa(initialReleaseDate.Year()))
}
if newDir != lastDir {
if err := d.mkdirAll(newDir, 0755); err != nil {
errorCh <- err
continue
}
lastDir = newDir
}
// Write advisory to file
filePath := filepath.Join(lastDir, filename)
// Write data to disk.
for _, x := range []struct {
p string
d []byte
}{
{filePath, data.Bytes()},
{filePath + ".sha256", s256Data},
{filePath + ".sha512", s512Data},
{filePath + ".asc", signData},
} {
if x.d != nil {
if err := os.WriteFile(x.p, x.d, 0644); err != nil {
errorCh <- err
continue nextAdvisory
}
}
}
stats.succeeded++
slog.Info("Written advisory", "path", filePath)
}
}
func (d *Downloader) mkdirAll(path string, perm os.FileMode) error {
d.mkdirMu.Lock()
defer d.mkdirMu.Unlock()
return os.MkdirAll(path, perm)
}
func (d *Downloader) checkSignature(data []byte, sign *crypto.PGPSignature) error {
pm := crypto.NewPlainMessage(data)
t := crypto.GetUnixTime()