Add abilities to aggregator to mirror and list

* Aggregator now checks every provider on whether its mirrored or listed. 
*Add the option to the docs.
* Clean up the example  toml file to still contain two mirrors and one example-lister.

Co-authored-by: Jan Höfelmeyer <Jan Höfelmeyer jhoefelmeyer@intevation.de>
Co-authored-by: Sascha L. Teichmann <sascha.teichmann@intevation.de>
Co-authored-by: Bernhard Reiter <bernhard@intevation.de>

cmd/csaf_aggregator/config.go

@@ -36,12 +36,13 @@ type provider struct {
 	Name   string `toml:"name"`
 	Domain string `toml:"domain"`
 	// Rate gives the provider specific rate limiting (see overall Rate).
-	Rate       *float64  `toml:"rate"`
+	Rate         *float64  `toml:"rate"`
-	Insecure   *bool     `toml:"insecure"`
+	Insecure     *bool     `toml:"insecure"`
-	Categories *[]string `toml:"categories"`
+	WriteIndices *bool     `toml:"write_indices"`
+	Categories   *[]string `toml:"categories"`
 	// ServiceDocument incidates if we should create a service.json document.
-	ServiceDocument *bool `toml:"create_service_document"`
+	ServiceDocument     *bool                    `toml:"create_service_document"`
-	WriteIndices    *bool `toml:"write_indices"`
+	AggregatoryCategory *csaf.AggregatorCategory `toml:"category"`
 }
 
 type config struct {
@@ -101,6 +102,26 @@ func (p *provider) writeIndices(c *config) bool {
 	return c.WriteIndices
 }
 
+func (p *provider) runAsMirror(c *config) bool {
+	if p.AggregatoryCategory != nil {
+		return *p.AggregatoryCategory == csaf.AggregatorAggregator
+	}
+	return c.runAsMirror()
+}
+
+// atLeastNMirrors checks if there are at least n mirrors configured.
+func (c *config) atLeastNMirrors(n int) bool {
+	var mirrors int
+	for _, p := range c.Providers {
+		if p.runAsMirror(c) {
+			if mirrors++; mirrors >= n {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // runAsMirror determines if the aggregator should run in mirror mode.
 func (c *config) runAsMirror() bool {
 	return c.Aggregator.Category != nil &&
@@ -184,6 +205,20 @@ func (c *config) checkProviders() error {
 	return nil
 }
 
+func (c *config) checkMirror() error {
+	if c.runAsMirror() {
+		if !c.AllowSingleProvider && !c.atLeastNMirrors(2) {
+			return errors.New("at least 2 providers need to be mirrored")
+		} else if c.AllowSingleProvider && !c.atLeastNMirrors(1) {
+			return errors.New("at least one provider must be mirrored")
+		}
+	} else if !c.AllowSingleProvider && c.atLeastNMirrors(1) {
+		return errors.New("found mirrors in a lister aggregator")
+	}
+
+	return nil
+}
+
 func (c *config) setDefaults() {
 	if c.Folder == "" {
 		c.Folder = defaultFolder
@@ -219,7 +254,11 @@ func (c *config) check() error {
 		return err
 	}
 
-	return c.checkProviders()
+	if err := c.checkProviders(); err != nil {
+		return err
+	}
+
+	return c.checkMirror()
 }
 
 func loadConfig(path string) (*config, error) {

cmd/csaf_aggregator/full.go

@@ -24,6 +24,7 @@ import (
 type fullJob struct {
 	provider           *provider
 	aggregatorProvider *csaf.AggregatorCSAFProvider
+	work               fullWorkFunc
 	err                error
 }
 
@@ -61,11 +62,7 @@ func (w *worker) setupProviderFull(provider *provider) error {
 type fullWorkFunc func(*worker) (*csaf.AggregatorCSAFProvider, error)
 
 // fullWork handles the treatment of providers concurrently.
-func (w *worker) fullWork(
+func (w *worker) fullWork(wg *sync.WaitGroup, jobs <-chan *fullJob) {
-	wg *sync.WaitGroup,
-	doWork fullWorkFunc,
-	jobs <-chan *fullJob,
-) {
 	defer wg.Done()
 
 	for j := range jobs {
@@ -73,16 +70,15 @@ func (w *worker) fullWork(
 			j.err = err
 			continue
 		}
-		j.aggregatorProvider, j.err = doWork(w)
+		j.aggregatorProvider, j.err = j.work(w)
 	}
 }
 
 // full performs the complete lister/download
 func (p *processor) full() error {
 
-	var doWork fullWorkFunc
-
 	if p.cfg.runAsMirror() {
+		log.Println("Running in aggregator mode")
 
 		// check if we need to setup a remote validator
 		if p.cfg.RemoteValidatorOptions != nil {
@@ -98,11 +94,7 @@ func (p *processor) full() error {
 				p.remoteValidator = nil
 			}()
 		}
-
-		doWork = (*worker).mirror
-		log.Println("Running in aggregator mode")
 	} else {
-		doWork = (*worker).lister
 		log.Println("Running in lister mode")
 	}
 
@@ -113,13 +105,22 @@ func (p *processor) full() error {
 	for i := 1; i <= p.cfg.Workers; i++ {
 		wg.Add(1)
 		w := newWorker(i, p)
-		go w.fullWork(&wg, doWork, queue)
+		go w.fullWork(&wg, queue)
 	}
 
 	jobs := make([]fullJob, len(p.cfg.Providers))
 
-	for i, p := range p.cfg.Providers {
+	for i, provider := range p.cfg.Providers {
-		jobs[i] = fullJob{provider: p}
+		var work fullWorkFunc
+		if provider.runAsMirror(p.cfg) {
+			work = (*worker).mirror
+		} else {
+			work = (*worker).lister
+		}
+		jobs[i] = fullJob{
+			provider: provider,
+			work:     work,
+		}
 		queue <- &jobs[i]
 	}
 	close(queue)

docs/csaf_aggregator.md

@@ -101,8 +101,14 @@ domain
 rate
 insecure
 write_indices
+category
 ```
 
+If you want an entry to be listed instead of mirrored
+in a `aggregator.category == "aggregator"` instance,
+set `category` to `lister` in the entry.
+Otherwise it is recommended to not set `category` for entries.
+
 #### Example config file
 <!-- MARKDOWN-AUTO-DOCS:START (CODE:src=../docs/examples/aggregator.toml) -->
 <!-- The below code snippet is automatically added from ../docs/examples/aggregator.toml -->
@@ -136,6 +142,7 @@ insecure = true
   create_service_document = true
 #  rate = 1.5
 #  insecure = true
+   category = "lister"
 
 [[providers]]
   name = "local-dev-provider2"
@@ -143,5 +150,14 @@ insecure = true
 #  rate = 1.2
 #  insecure = true
   write_indices = true
+  category = "aggregator"
+  
+[[providers]]
+  name = "local-dev-provider3"
+  domain = "localhost"
+#  rate = 1.8
+#  insecure = true
+  write_indices = true
+  category = "aggregator"
 ```
 <!-- MARKDOWN-AUTO-DOCS:END -->

docs/examples/aggregator.toml

@@ -14,6 +14,8 @@ insecure = true
 # allow_single_provider = true
 
 [aggregator]
+  # Set if this instance shall be a mirror (aka `aggregator`) or a `lister`.
+  # This determines the default value for the entries in [[provider]].
   category = "aggregator"
   name = "Example Development CSAF Aggregator"
   contact_details = "some @ somewhere"
@@ -34,3 +36,13 @@ insecure = true
 #  rate = 1.2
 #  insecure = true
   write_indices = true
+
+[[providers]]
+  name = "local-dev-provider3"
+  domain = "localhost"
+#  rate = 1.8
+#  insecure = true
+  write_indices = true
+  # If aggregator.category == "aggreator", set for an entry that should
+  # be listed in addition:
+  category = "lister"