Add ignore patterns to aggreagtor.

cmd/csaf_aggregator/config.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 	"github.com/csaf-poc/csaf_distribution/v2/csaf"
+	"github.com/csaf-poc/csaf_distribution/v2/internal/filter"
 	"github.com/csaf-poc/csaf_distribution/v2/internal/options"
 	"github.com/csaf-poc/csaf_distribution/v2/util"
 	"golang.org/x/time/rate"
@@ -48,6 +49,10 @@ type provider struct {
 
 	// UpdateInterval is as the mandatory `update_interval` if this is a publisher.
 	UpdateInterval *string `toml:"update_interval"`
+
+	// IgnorePattern is a list of patterns of advisory URLs to be ignored.
+	IgnorePattern []string `toml:"ignorepattern"`
+	ignorePattern filter.PatternMatcher
 }
 
 type config struct {
@@ -90,6 +95,10 @@ type config struct {
 	// 'update_interval'.
 	UpdateInterval *string `toml:"update_interval"`
 
+	// IgnorePattern is a list of patterns of advisory URLs to be ignored.
+	IgnorePattern []string `toml:"ignorepattern"`
+	ignorePattern filter.PatternMatcher
+
 	Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"`
 
 	keyMu  sync.Mutex
@@ -128,6 +137,11 @@ func (c *config) tooOldForInterims() func(time.Time) bool {
 	return func(t time.Time) bool { return t.Before(from) }
 }
 
+// ignoreFile returns true if the given URL should not be downloaded.
+func (p *provider) ignoreURL(u string, c *config) bool {
+	return p.ignorePattern.Matches(u) || c.ignorePattern.Matches(u)
+}
+
 // updateInterval returns the update interval of a publisher.
 func (p *provider) updateInterval(c *config) string {
 	if p.UpdateInterval != nil {
@@ -307,12 +321,44 @@ func (c *config) setDefaults() {
 	}
 }
 
+// compileIgnorePatterns compiles the configured patterns to be ignored.
+func (p *provider) compileIgnorePatterns() error {
+	pm, err := filter.NewPatternMatcher(p.IgnorePattern)
+	if err != nil {
+		return err
+	}
+	p.ignorePattern = pm
+	return nil
+}
+
+// compileIgnorePatterns compiles the configured patterns to be ignored.
+func (c *config) compileIgnorePatterns() error {
+	// Compile the top level patterns.
+	pm, err := filter.NewPatternMatcher(c.IgnorePattern)
+	if err != nil {
+		return err
+	}
+	c.ignorePattern = pm
+	// Compile the patterns of the providers.
+	for _, p := range c.Providers {
+		if err := p.compileIgnorePatterns(); err != nil {
+			return fmt.Errorf("invalid ignore patterns for %q: %w", p.Name, err)
+		}
+	}
+	return nil
+}
+
 // prepare prepares internal state of a loaded configuration.
 func (c *config) prepare() error {
+
 	if len(c.Providers) == 0 {
 		return errors.New("no providers given in configuration")
 	}
 
+	if err := c.compileIgnorePatterns(); err != nil {
+		return err
+	}
+
 	if err := c.Aggregator.Validate(); err != nil {
 		return err
 	}

cmd/csaf_aggregator/mirror.go

@@ -500,6 +500,14 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 			continue
 		}
 
+		// Should we ignore this advisory?
+		if w.provider.ignoreURL(file.URL(), w.processor.cfg) {
+			if w.processor.cfg.Verbose {
+				log.Printf("Ignoring %s: %q\n", w.provider.Name, file.URL())
+			}
+			continue
+		}
+
 		// Ignore not conforming filenames.
 		filename := filepath.Base(u.Path)
 		if !util.ConformingFileName(filename) {

docs/csaf_aggregator.md

@@ -94,6 +94,7 @@ lock_file               // path to lockfile, to stop other instances if one is n
 interim_years           // limiting the years for which interim documents are searched (default 0)
 verbose                 // print more diagnostic output, e.g. https requests (default false)
 allow_single_provider   // debugging option (default false)
+ignorepattern           // patterns of advisory URLs to be ignored
 ```
 
 Next we have two TOML _tables_:

docs/examples/aggregator.toml

@@ -48,3 +48,4 @@ insecure = true
   # If aggregator.category == "aggreator", set for an entry that should
   # be listed in addition:
   category = "lister"
+# ignorepattern = [".*white.*", ".*red.*"]