Improve download handler interface

cmd/csaf_downloader/main.go

@@ -11,6 +11,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"log/slog"
 	"os"
 	"os/signal"
@@ -19,9 +20,12 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
+	"github.com/csaf-poc/csaf_distribution/v3/csaf"
 	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
 	"github.com/csaf-poc/csaf_distribution/v3/lib/downloader"
+	"github.com/csaf-poc/csaf_distribution/v3/util"
 )
 
 // failedForwardDir is the name of the special sub folder
@@ -79,6 +83,35 @@ func mkdirAll(path string, perm os.FileMode) error {
 	return os.MkdirAll(path, perm)
 }
 
+func extractInitialReleaseDate(doc any) time.Time {
+	var initialReleaseDate time.Time
+	dateExtract := util.TimeMatcher(&initialReleaseDate, time.RFC3339)
+
+	eval := util.NewPathEval()
+
+	if err := eval.Extract(
+		`$.document.tracking.initial_release_date`, dateExtract, false, doc,
+	); err != nil {
+		slog.Warn("Cannot extract initial_release_date from advisory")
+		initialReleaseDate = time.Now()
+	}
+	initialReleaseDate = initialReleaseDate.UTC()
+	return initialReleaseDate
+}
+
+func extractTLP(doc any) csaf.TLPLabel {
+	eval := util.NewPathEval()
+	labelString, err := eval.Eval(`$.document.distribution.tlp.label`, doc)
+	if err != nil {
+		return csaf.TLPLabelUnlabeled
+	}
+	label, ok := labelString.(string)
+	if !ok {
+		return csaf.TLPLabelUnlabeled
+	}
+	return csaf.TLPLabel(label)
+}
+
 func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error {
 	return func(d downloader.DownloadedDocument) error {
 		if cfg.NoStore {
@@ -98,13 +131,22 @@ func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error {
 			newDir = cfg.Directory
 		}
 
-		lower := strings.ToLower(string(d.Label))
+		var doc any
+		if err := json.Unmarshal(d.Data, &doc); err != nil {
+			slog.Error("Could not parse json document", "err", err)
+			return nil
+		}
+
+		initialReleaseDate := extractInitialReleaseDate(doc)
+		label := extractTLP(doc)
+
+		lower := strings.ToLower(string(label))
 
 		// Do we have a configured destination folder?
 		if cfg.Folder != "" {
 			newDir = path.Join(newDir, cfg.Folder)
 		} else {
-			newDir = path.Join(newDir, lower, strconv.Itoa(d.InitialReleaseDate.Year()))
+			newDir = path.Join(newDir, lower, strconv.Itoa(initialReleaseDate.Year()))
 		}
 
 		if newDir != lastDir {
@@ -121,9 +163,9 @@ func downloadHandler(cfg *config) func(d downloader.DownloadedDocument) error {
 			p string
 			d []byte
 		}{
-			{filePath, d.Data.Bytes()},
+			{filePath, d.Data},
-			{filePath + ".sha256", d.S256Data},
+			{filePath + ".sha256", d.SHA256},
-			{filePath + ".sha512", d.S512Data},
+			{filePath + ".sha512", d.SHA512},
 			{filePath + ".asc", d.SignData},
 		} {
 			if x.d != nil {
@@ -157,8 +199,8 @@ func storeFailedAdvisory(cfg *config) func(filename, doc, sha256, sha512 string)
 			{filename + ".sha512", sha512},
 		} {
 			if len(x.d) != 0 {
-				path := filepath.Join(dir, x.p)
+				p := filepath.Join(dir, x.p)
-				if err := os.WriteFile(path, []byte(x.d), 0644); err != nil {
+				if err := os.WriteFile(p, []byte(x.d), 0644); err != nil {
 					return err
 				}
 			}

lib/downloader/downloader.go

@@ -38,6 +38,7 @@ import (
 type Downloader struct {
 	cfg       *Config
 	keys      *crypto.KeyRing
+	eval      *util.PathEval
 	validator csaf.RemoteValidator
 	Forwarder *Forwarder
 	mkdirMu   sync.Mutex
@@ -47,14 +48,12 @@ type Downloader struct {
 
 // DownloadedDocument contains the document data with additional metadata.
 type DownloadedDocument struct {
-	Data               bytes.Buffer
+	Data      []byte
-	S256Data           []byte
+	SHA256    []byte
-	S512Data           []byte
+	SHA512    []byte
 	SignData  []byte
-	InitialReleaseDate time.Time
 	Filename  string
 	ValStatus ValidationStatus
-	Label              csaf.TLPLabel
 }
 
 // failedValidationDir is the name of the sub folder
@@ -64,7 +63,6 @@ const failedValidationDir = "failed_validation"
 
 // NewDownloader constructs a new downloader given the configuration.
 func NewDownloader(cfg *Config) (*Downloader, error) {
-
 	var validator csaf.RemoteValidator
 
 	if cfg.RemoteValidator != "" {
@@ -117,7 +115,6 @@ func logRedirect(logger *slog.Logger) func(req *http.Request, via []*http.Reques
 }
 
 func (d *Downloader) httpClient() util.Client {
-
 	hClient := http.Client{}
 
 	if d.cfg.verbose() {
@@ -258,16 +255,14 @@ func (d *Downloader) download(ctx context.Context, domain string) error {
 	}
 
 	return afp.Process(func(label csaf.TLPLabel, files []csaf.AdvisoryFile) error {
-		return d.downloadFiles(ctx, label, files)
+		return d.downloadFiles(ctx, files)
 	})
 }
 
 func (d *Downloader) downloadFiles(
 	ctx context.Context,
-	label csaf.TLPLabel,
 	files []csaf.AdvisoryFile,
 ) error {
-
 	var (
 		advisoryCh = make(chan csaf.AdvisoryFile)
 		errorCh    = make(chan error)
@@ -291,7 +286,7 @@ func (d *Downloader) downloadFiles(
 
 	for i := 0; i < n; i++ {
 		wg.Add(1)
-		go d.downloadWorker(ctx, &wg, label, advisoryCh, errorCh)
+		go d.downloadWorker(ctx, &wg, advisoryCh, errorCh)
 	}
 
 allFiles:
@@ -370,7 +365,6 @@ func (d *Downloader) loadOpenPGPKeys(
 			defer res.Body.Close()
 			return crypto.NewKeyFromArmoredReader(res.Body)
 		}()
-
 		if err != nil {
 			d.cfg.Logger.Warn(
 				"Reading public OpenPGP key failed",
@@ -425,7 +419,6 @@ func (d *Downloader) logValidationIssues(url string, errors []string, err error)
 func (d *Downloader) downloadWorker(
 	ctx context.Context,
 	wg *sync.WaitGroup,
-	label csaf.TLPLabel,
 	files <-chan csaf.AdvisoryFile,
 	errorCh chan<- error,
 ) {
@@ -434,8 +427,6 @@ func (d *Downloader) downloadWorker(
 	var (
 		client = d.httpClient()
 		data   bytes.Buffer
-		initialReleaseDate time.Time
-		dateExtract        = util.TimeMatcher(&initialReleaseDate, time.RFC3339)
 		stats  = stats{}
 		expr               = util.NewPathEval()
 	)
@@ -659,24 +650,14 @@ nextAdvisory:
 				string(s256Data),
 				string(s512Data))
 		}
-		if err := expr.Extract(
-			`$.document.tracking.initial_release_date`, dateExtract, false, doc,
-		); err != nil {
-			slog.Warn("Cannot extract initial_release_date from advisory",
-				"url", file.URL())
-			initialReleaseDate = time.Now()
-		}
-		initialReleaseDate = initialReleaseDate.UTC()
 
 		download := DownloadedDocument{
-			Data:               data,
+			Data:      data.Bytes(),
-			S256Data:           s256Data,
+			SHA256:    s256Data,
-			S512Data:           s512Data,
+			SHA512:    s512Data,
 			SignData:  signData,
-			InitialReleaseDate: initialReleaseDate,
 			Filename:  filename,
 			ValStatus: valStatus,
-			Label:              label,
 		}
 
 		err = d.cfg.DownloadHandler(download)