From 7bafb210cf6d34a802dc943b9be7722dd00116cd Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Thu, 30 Jun 2022 13:54:51 +0200
Subject: [PATCH] Add category support for aggregator

---
 cmd/csaf_aggregator/config.go    |  6 ++--
 cmd/csaf_aggregator/indices.go   | 22 ++++++++++++++
 cmd/csaf_aggregator/mirror.go    | 50 ++++++++++++++++++++++++++++++++
 cmd/csaf_aggregator/processor.go | 13 +++++----
 4 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go
index af5d5a0..387a7f9 100644
--- a/cmd/csaf_aggregator/config.go
+++ b/cmd/csaf_aggregator/config.go
@@ -36,8 +36,9 @@ type provider struct {
 	Name   string `toml:"name"`
 	Domain string `toml:"domain"`
 	// Rate gives the provider specific rate limiting (see overall Rate).
-	Rate     *float64 `toml:"rate"`
-	Insecure *bool    `toml:"insecure"`
+	Rate       *float64  `toml:"rate"`
+	Insecure   *bool     `toml:"insecure"`
+	Categories *[]string `toml:"categories"`
 }
 
 type config struct {
@@ -50,6 +51,7 @@ type config struct {
 	// Rate gives the average upper limit of https operations per second.
 	Rate                *float64            `toml:"rate"`
 	Insecure            *bool               `toml:"insecure"`
+	Categories          *[]string           `toml:"categories"`
 	Aggregator          csaf.AggregatorInfo `toml:"aggregator"`
 	Providers           []*provider         `toml:"providers"`
 	OpenPGPPrivateKey   string              `toml:"openpgp_private_key"`
diff --git a/cmd/csaf_aggregator/indices.go b/cmd/csaf_aggregator/indices.go
index 71bab71..40c2a01 100644
--- a/cmd/csaf_aggregator/indices.go
+++ b/cmd/csaf_aggregator/indices.go
@@ -209,6 +209,25 @@ func (w *worker) writeROLIE(label string, summaries []summary) error {
 	return util.WriteToFile(path, rolie)
 }
 
+func (w *worker) writeCategories(label string) error {
+	categories := w.categories[label]
+	if len(categories) == 0 {
+		return nil
+	}
+	cats := make([]string, len(categories))
+	var i int
+	for cat := range categories {
+		cats[i] = cat
+		i++
+	}
+	rcd := csaf.NewROLIECategoryDocument(cats...)
+
+	labelFolder := strings.ToLower(label)
+	fname := "category-" + labelFolder + ".json"
+	path := filepath.Join(w.dir, labelFolder, fname)
+	return util.WriteToFile(path, rcd)
+}
+
 func (w *worker) writeIndices() error {
 
 	if len(w.summaries) == 0 || w.dir == "" {
@@ -229,6 +248,9 @@ func (w *worker) writeIndices() error {
 		if err := w.writeROLIE(label, summaries); err != nil {
 			return err
 		}
+		if err := w.writeCategories(label); err != nil {
+			return err
+		}
 	}
 
 	return nil
diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go
index a224147..8bcef34 100644
--- a/cmd/csaf_aggregator/mirror.go
+++ b/cmd/csaf_aggregator/mirror.go
@@ -64,6 +64,9 @@ func (w *worker) mirrorInternal() (*csaf.AggregatorCSAFProvider, error) {
 	// Collecting the summaries of the advisories.
 	w.summaries = make(map[string][]summary)
 
+	// Collecting the categories per label.
+	w.categories = make(map[string]map[string]bool)
+
 	base, err := url.Parse(w.loc)
 	if err != nil {
 		return nil, err
@@ -413,6 +416,47 @@ func (w *worker) sign(data []byte) (string, error) {
 		sig.Data, constants.PGPSignatureHeader, "", "")
 }
 
+func (w *worker) extractCategories(label string, advisory interface{}) error {
+
+	// use provider or global categories
+	var categories []string
+	if w.provider.Categories != nil {
+		categories = *w.provider.Categories
+	} else if w.processor.cfg.Categories != nil {
+		categories = *w.processor.cfg.Categories
+	}
+
+	// Nothing to do.
+	if len(categories) == 0 {
+		return nil
+	}
+
+	cats := w.categories[label]
+	if cats == nil {
+		cats = make(map[string]bool)
+		w.categories[label] = cats
+	}
+
+	var result string
+	matcher := util.StringMatcher(&result)
+
+	const exprPrefix = "expr:"
+
+	for _, cat := range categories {
+		if strings.HasPrefix(cat, exprPrefix) {
+			expr := cat[len(exprPrefix):]
+			if err := w.expr.Extract(expr, matcher, true, advisory); err != nil {
+				return err
+			}
+			cats[result] = true
+		} else { // Normal
+			cats[cat] = true
+		}
+	}
+
+	return nil
+}
+
 func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile) error {
 	label := strings.ToLower(string(tlpLabel))
 
@@ -489,6 +533,12 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 			log.Printf("error: %s: %v\n", file, err)
 			continue
 		}
+
+		if err := w.extractCategories(label, advisory); err != nil {
+			log.Printf("error: %s: %v\n", file, err)
+			continue
+		}
+
 		summaries = append(summaries, summary{
 			filename: filename,
 			summary:  sum,
diff --git a/cmd/csaf_aggregator/processor.go b/cmd/csaf_aggregator/processor.go
index cd570d3..5ae4f66 100644
--- a/cmd/csaf_aggregator/processor.go
+++ b/cmd/csaf_aggregator/processor.go
@@ -40,12 +40,13 @@ type worker struct {
 	expr     *util.PathEval
 	signRing *crypto.KeyRing
 
-	client           util.Client          // client per provider
-	provider         *provider            // current provider
-	metadataProvider interface{}          // current metadata provider
-	loc              string               // URL of current provider-metadata.json
-	dir              string               // Directory to store data to.
-	summaries        map[string][]summary // the summaries of the advisories.
+	client           util.Client                // client per provider
+	provider         *provider                  // current provider
+	metadataProvider interface{}                // current metadata provider
+	loc              string                     // URL of current provider-metadata.json
+	dir              string                     // Directory to store data to.
+	summaries        map[string][]summary       // the summaries of the advisories.
+	categories       map[string]map[string]bool // the categories per label.
 }
 
 func newWorker(num int, processor *processor) *worker {