diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 3270a88..f0778ee 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -417,6 +417,321 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error) } } +// downloadContext stores the common context of a downloader. +type downloadContext struct { + d *downloader + client util.Client + data bytes.Buffer + lastDir string + initialReleaseDate time.Time + dateExtract func(any) error + lower string + stats stats + expr *util.PathEval +} + +func newDownloadContext(d *downloader, label csaf.TLPLabel) *downloadContext { + dc := &downloadContext{ + d: d, + client: d.httpClient(), + lower: strings.ToLower(string(label)), + expr: util.NewPathEval(), + } + dc.dateExtract = util.TimeMatcher(&dc.initialReleaseDate, time.RFC3339) + return dc +} + +func (dc *downloadContext) downloadAdvisory( + file csaf.AdvisoryFile, + errorCh chan<- error, +) error { + u, err := url.Parse(file.URL()) + if err != nil { + dc.stats.downloadFailed++ + slog.Warn("Ignoring invalid URL", + "url", file.URL(), + "error", err) + return nil + } + + if dc.d.cfg.ignoreURL(file.URL()) { + slog.Debug("Ignoring URL", "url", file.URL()) + return nil + } + + // Ignore not conforming filenames. + filename := filepath.Base(u.Path) + if !util.ConformingFileName(filename) { + dc.stats.filenameFailed++ + slog.Warn("Ignoring none conforming filename", + "filename", filename) + return nil + } + + resp, err := dc.client.Get(file.URL()) + if err != nil { + dc.stats.downloadFailed++ + slog.Warn("Cannot GET", + "url", file.URL(), + "error", err) + return nil + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + dc.stats.downloadFailed++ + slog.Warn("Cannot load", + "url", file.URL(), + "status", resp.Status, + "status_code", resp.StatusCode) + return nil + } + + // Warn if we do not get JSON. + if ct := resp.Header.Get("Content-Type"); ct != "application/json" { + slog.Warn("Content type is not 'application/json'", + "url", file.URL(), + "content_type", ct) + } + + var ( + writers []io.Writer + s256, s512 hash.Hash + s256Data, s512Data []byte + remoteSHA256, remoteSHA512 []byte + signData []byte + ) + + hashToFetch := []hashFetchInfo{} + if file.SHA512URL() != "" { + hashToFetch = append(hashToFetch, hashFetchInfo{ + url: file.SHA512URL(), + warn: true, + hashType: algSha512, + preferred: strings.EqualFold(string(dc.d.cfg.PreferredHash), string(algSha512)), + }) + } else { + slog.Info("SHA512 not present") + } + if file.SHA256URL() != "" { + hashToFetch = append(hashToFetch, hashFetchInfo{ + url: file.SHA256URL(), + warn: true, + hashType: algSha256, + preferred: strings.EqualFold(string(dc.d.cfg.PreferredHash), string(algSha256)), + }) + } else { + slog.Info("SHA256 not present") + } + if file.IsDirectory() { + for i := range hashToFetch { + hashToFetch[i].warn = false + } + } + + remoteSHA256, s256Data, remoteSHA512, s512Data = loadHashes(dc.client, hashToFetch) + if remoteSHA512 != nil { + s512 = sha512.New() + writers = append(writers, s512) + } + if remoteSHA256 != nil { + s256 = sha256.New() + writers = append(writers, s256) + } + + // Remember the data as we need to store it to file later. + dc.data.Reset() + writers = append(writers, &dc.data) + + // Download the advisory and hash it. + hasher := io.MultiWriter(writers...) + + var doc any + + tee := io.TeeReader(resp.Body, hasher) + + if err := json.NewDecoder(tee).Decode(&doc); err != nil { + dc.stats.downloadFailed++ + slog.Warn("Downloading failed", + "url", file.URL(), + "error", err) + return nil + } + + // Compare the checksums. + s256Check := func() error { + if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) { + dc.stats.sha256Failed++ + return fmt.Errorf("SHA256 checksum of %s does not match", file.URL()) + } + return nil + } + + s512Check := func() error { + if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) { + dc.stats.sha512Failed++ + return fmt.Errorf("SHA512 checksum of %s does not match", file.URL()) + } + return nil + } + + // Validate OpenPGP signature. + keysCheck := func() error { + // Only check signature if we have loaded keys. + if dc.d.keys == nil { + return nil + } + var sign *crypto.PGPSignature + sign, signData, err = loadSignature(dc.client, file.SignURL()) + if err != nil { + slog.Warn("Downloading signature failed", + "url", file.SignURL(), + "error", err) + } + if sign != nil { + if err := dc.d.checkSignature(dc.data.Bytes(), sign); err != nil { + if !dc.d.cfg.IgnoreSignatureCheck { + dc.stats.signatureFailed++ + return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err) + } + } + } + return nil + } + + // Validate against CSAF schema. + schemaCheck := func() error { + if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 { + dc.stats.schemaFailed++ + dc.d.logValidationIssues(file.URL(), errors, err) + return fmt.Errorf("schema validation for %q failed", file.URL()) + } + return nil + } + + // Validate if filename is conforming. + filenameCheck := func() error { + if err := util.IDMatchesFilename(dc.expr, doc, filename); err != nil { + dc.stats.filenameFailed++ + return fmt.Errorf("filename not conforming %s: %s", file.URL(), err) + } + return nil + } + + // Validate against remote validator. + remoteValidatorCheck := func() error { + if dc.d.validator == nil { + return nil + } + rvr, err := dc.d.validator.Validate(doc) + if err != nil { + errorCh <- fmt.Errorf( + "calling remote validator on %q failed: %w", + file.URL(), err) + return nil + } + if !rvr.Valid { + dc.stats.remoteFailed++ + return fmt.Errorf("remote validation of %q failed", file.URL()) + } + return nil + } + + // Run all the validations. + valStatus := notValidatedValidationStatus + for _, check := range []func() error{ + s256Check, + s512Check, + keysCheck, + schemaCheck, + filenameCheck, + remoteValidatorCheck, + } { + if err := check(); err != nil { + slog.Error("Validation check failed", "error", err) + valStatus.update(invalidValidationStatus) + if dc.d.cfg.ValidationMode == validationStrict { + return nil + } + } + } + valStatus.update(validValidationStatus) + + // Send to forwarder + if dc.d.forwarder != nil { + dc.d.forwarder.forward( + filename, dc.data.String(), + valStatus, + string(s256Data), + string(s512Data)) + } + + if dc.d.cfg.NoStore { + // Do not write locally. + if valStatus == validValidationStatus { + dc.stats.succeeded++ + } + return nil + } + + if err := dc.expr.Extract( + `$.document.tracking.initial_release_date`, dc.dateExtract, false, doc, + ); err != nil { + slog.Warn("Cannot extract initial_release_date from advisory", + "url", file.URL()) + dc.initialReleaseDate = time.Now() + } + dc.initialReleaseDate = dc.initialReleaseDate.UTC() + + // Advisories that failed validation are stored in a special folder. + var newDir string + if valStatus != validValidationStatus { + newDir = path.Join(dc.d.cfg.Directory, failedValidationDir) + } else { + newDir = dc.d.cfg.Directory + } + + // Do we have a configured destination folder? + if dc.d.cfg.Folder != "" { + newDir = path.Join(newDir, dc.d.cfg.Folder) + } else { + newDir = path.Join(newDir, dc.lower, strconv.Itoa(dc.initialReleaseDate.Year())) + } + + if newDir != dc.lastDir { + if err := dc.d.mkdirAll(newDir, 0755); err != nil { + errorCh <- err + return nil + } + dc.lastDir = newDir + } + + // Write advisory to file + path := filepath.Join(dc.lastDir, filename) + + // Write data to disk. + for _, x := range []struct { + p string + d []byte + }{ + {path, dc.data.Bytes()}, + {path + ".sha256", s256Data}, + {path + ".sha512", s512Data}, + {path + ".asc", signData}, + } { + if x.d != nil { + if err := os.WriteFile(x.p, x.d, 0644); err != nil { + errorCh <- err + return nil + } + } + } + + dc.stats.succeeded++ + slog.Info("Written advisory", "path", path) + return nil +} + func (d *downloader) downloadWorker( ctx context.Context, wg *sync.WaitGroup, @@ -426,21 +741,11 @@ func (d *downloader) downloadWorker( ) { defer wg.Done() - var ( - client = d.httpClient() - data bytes.Buffer - lastDir string - initialReleaseDate time.Time - dateExtract = util.TimeMatcher(&initialReleaseDate, time.RFC3339) - lower = strings.ToLower(string(label)) - stats = stats{} - expr = util.NewPathEval() - ) + dc := newDownloadContext(d, label) // Add collected stats back to total. - defer d.addStats(&stats) + defer d.addStats(&dc.stats) -nextAdvisory: for { var file csaf.AdvisoryFile var ok bool @@ -452,292 +757,10 @@ nextAdvisory: case <-ctx.Done(): return } - - u, err := url.Parse(file.URL()) - if err != nil { - stats.downloadFailed++ - slog.Warn("Ignoring invalid URL", - "url", file.URL(), - "error", err) - continue + if err := dc.downloadAdvisory(file, errorCh); err != nil { + slog.Error("download terminated", "error", err) + return } - - if d.cfg.ignoreURL(file.URL()) { - slog.Debug("Ignoring URL", "url", file.URL()) - continue - } - - // Ignore not conforming filenames. - filename := filepath.Base(u.Path) - if !util.ConformingFileName(filename) { - stats.filenameFailed++ - slog.Warn("Ignoring none conforming filename", - "filename", filename) - continue - } - - resp, err := client.Get(file.URL()) - if err != nil { - stats.downloadFailed++ - slog.Warn("Cannot GET", - "url", file.URL(), - "error", err) - continue - } - - if resp.StatusCode != http.StatusOK { - stats.downloadFailed++ - slog.Warn("Cannot load", - "url", file.URL(), - "status", resp.Status, - "status_code", resp.StatusCode) - continue - } - - // Warn if we do not get JSON. - if ct := resp.Header.Get("Content-Type"); ct != "application/json" { - slog.Warn("Content type is not 'application/json'", - "url", file.URL(), - "content_type", ct) - } - - var ( - writers []io.Writer - s256, s512 hash.Hash - s256Data, s512Data []byte - remoteSHA256, remoteSHA512 []byte - signData []byte - ) - - hashToFetch := []hashFetchInfo{} - if file.SHA512URL() != "" { - hashToFetch = append(hashToFetch, hashFetchInfo{ - url: file.SHA512URL(), - warn: true, - hashType: algSha512, - preferred: strings.EqualFold(string(d.cfg.PreferredHash), string(algSha512)), - }) - } else { - slog.Info("SHA512 not present") - } - if file.SHA256URL() != "" { - hashToFetch = append(hashToFetch, hashFetchInfo{ - url: file.SHA256URL(), - warn: true, - hashType: algSha256, - preferred: strings.EqualFold(string(d.cfg.PreferredHash), string(algSha256)), - }) - } else { - slog.Info("SHA256 not present") - } - if file.IsDirectory() { - for i := range hashToFetch { - hashToFetch[i].warn = false - } - } - - remoteSHA256, s256Data, remoteSHA512, s512Data = loadHashes(client, hashToFetch) - if remoteSHA512 != nil { - s512 = sha512.New() - writers = append(writers, s512) - } - if remoteSHA256 != nil { - s256 = sha256.New() - writers = append(writers, s256) - } - - // Remember the data as we need to store it to file later. - data.Reset() - writers = append(writers, &data) - - // Download the advisory and hash it. - hasher := io.MultiWriter(writers...) - - var doc any - - if err := func() error { - defer resp.Body.Close() - tee := io.TeeReader(resp.Body, hasher) - return json.NewDecoder(tee).Decode(&doc) - }(); err != nil { - stats.downloadFailed++ - slog.Warn("Downloading failed", - "url", file.URL(), - "error", err) - continue - } - - // Compare the checksums. - s256Check := func() error { - if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) { - stats.sha256Failed++ - return fmt.Errorf("SHA256 checksum of %s does not match", file.URL()) - } - return nil - } - - s512Check := func() error { - if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) { - stats.sha512Failed++ - return fmt.Errorf("SHA512 checksum of %s does not match", file.URL()) - } - return nil - } - - // Validate OpenPGP signature. - keysCheck := func() error { - // Only check signature if we have loaded keys. - if d.keys == nil { - return nil - } - var sign *crypto.PGPSignature - sign, signData, err = loadSignature(client, file.SignURL()) - if err != nil { - slog.Warn("Downloading signature failed", - "url", file.SignURL(), - "error", err) - } - if sign != nil { - if err := d.checkSignature(data.Bytes(), sign); err != nil { - if !d.cfg.IgnoreSignatureCheck { - stats.signatureFailed++ - return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err) - } - } - } - return nil - } - - // Validate against CSAF schema. - schemaCheck := func() error { - if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 { - stats.schemaFailed++ - d.logValidationIssues(file.URL(), errors, err) - return fmt.Errorf("schema validation for %q failed", file.URL()) - } - return nil - } - - // Validate if filename is conforming. - filenameCheck := func() error { - if err := util.IDMatchesFilename(expr, doc, filename); err != nil { - stats.filenameFailed++ - return fmt.Errorf("filename not conforming %s: %s", file.URL(), err) - } - return nil - } - - // Validate against remote validator. - remoteValidatorCheck := func() error { - if d.validator == nil { - return nil - } - rvr, err := d.validator.Validate(doc) - if err != nil { - errorCh <- fmt.Errorf( - "calling remote validator on %q failed: %w", - file.URL(), err) - return nil - } - if !rvr.Valid { - stats.remoteFailed++ - return fmt.Errorf("remote validation of %q failed", file.URL()) - } - return nil - } - - // Run all the validations. - valStatus := notValidatedValidationStatus - for _, check := range []func() error{ - s256Check, - s512Check, - keysCheck, - schemaCheck, - filenameCheck, - remoteValidatorCheck, - } { - if err := check(); err != nil { - slog.Error("Validation check failed", "error", err) - valStatus.update(invalidValidationStatus) - if d.cfg.ValidationMode == validationStrict { - continue nextAdvisory - } - } - } - valStatus.update(validValidationStatus) - - // Send to forwarder - if d.forwarder != nil { - d.forwarder.forward( - filename, data.String(), - valStatus, - string(s256Data), - string(s512Data)) - } - - if d.cfg.NoStore { - // Do not write locally. - if valStatus == validValidationStatus { - stats.succeeded++ - } - continue - } - - if err := expr.Extract( - `$.document.tracking.initial_release_date`, dateExtract, false, doc, - ); err != nil { - slog.Warn("Cannot extract initial_release_date from advisory", - "url", file.URL()) - initialReleaseDate = time.Now() - } - initialReleaseDate = initialReleaseDate.UTC() - - // Advisories that failed validation are stored in a special folder. - var newDir string - if valStatus != validValidationStatus { - newDir = path.Join(d.cfg.Directory, failedValidationDir) - } else { - newDir = d.cfg.Directory - } - - // Do we have a configured destination folder? - if d.cfg.Folder != "" { - newDir = path.Join(newDir, d.cfg.Folder) - } else { - newDir = path.Join(newDir, lower, strconv.Itoa(initialReleaseDate.Year())) - } - - if newDir != lastDir { - if err := d.mkdirAll(newDir, 0755); err != nil { - errorCh <- err - continue - } - lastDir = newDir - } - - // Write advisory to file - path := filepath.Join(lastDir, filename) - - // Write data to disk. - for _, x := range []struct { - p string - d []byte - }{ - {path, data.Bytes()}, - {path + ".sha256", s256Data}, - {path + ".sha512", s512Data}, - {path + ".asc", signData}, - } { - if x.d != nil { - if err := os.WriteFile(x.p, x.d, 0644); err != nil { - errorCh <- err - continue nextAdvisory - } - } - } - - stats.succeeded++ - slog.Info("Written advisory", "path", path) } }