From 3769f1d33868a96f16637424e6d13b3e8ad6d83a Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer <107021473+JanHoefelmeyer@users.noreply.github.com> Date: Thu, 21 Jul 2022 17:59:58 +0200 Subject: [PATCH] Add abilities to aggregator to mirror and list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Aggregator now checks every provider on whether its mirrored or listed. *Add the option to the docs. * Clean up the example toml file to still contain two mirrors and one example-lister. Co-authored-by: Jan Höfelmeyer Co-authored-by: Sascha L. Teichmann Co-authored-by: Bernhard Reiter --- cmd/csaf_aggregator/config.go | 51 ++++++++++++++++++++++++++++++----- cmd/csaf_aggregator/full.go | 31 ++++++++++----------- docs/csaf_aggregator.md | 16 +++++++++++ docs/examples/aggregator.toml | 12 +++++++++ 4 files changed, 89 insertions(+), 21 deletions(-) diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go index ef97a60..55f390c 100644 --- a/cmd/csaf_aggregator/config.go +++ b/cmd/csaf_aggregator/config.go @@ -36,12 +36,13 @@ type provider struct { Name string `toml:"name"` Domain string `toml:"domain"` // Rate gives the provider specific rate limiting (see overall Rate). - Rate *float64 `toml:"rate"` - Insecure *bool `toml:"insecure"` - Categories *[]string `toml:"categories"` + Rate *float64 `toml:"rate"` + Insecure *bool `toml:"insecure"` + WriteIndices *bool `toml:"write_indices"` + Categories *[]string `toml:"categories"` // ServiceDocument incidates if we should create a service.json document. - ServiceDocument *bool `toml:"create_service_document"` - WriteIndices *bool `toml:"write_indices"` + ServiceDocument *bool `toml:"create_service_document"` + AggregatoryCategory *csaf.AggregatorCategory `toml:"category"` } type config struct { @@ -101,6 +102,26 @@ func (p *provider) writeIndices(c *config) bool { return c.WriteIndices } +func (p *provider) runAsMirror(c *config) bool { + if p.AggregatoryCategory != nil { + return *p.AggregatoryCategory == csaf.AggregatorAggregator + } + return c.runAsMirror() +} + +// atLeastNMirrors checks if there are at least n mirrors configured. +func (c *config) atLeastNMirrors(n int) bool { + var mirrors int + for _, p := range c.Providers { + if p.runAsMirror(c) { + if mirrors++; mirrors >= n { + return true + } + } + } + return false +} + // runAsMirror determines if the aggregator should run in mirror mode. func (c *config) runAsMirror() bool { return c.Aggregator.Category != nil && @@ -184,6 +205,20 @@ func (c *config) checkProviders() error { return nil } +func (c *config) checkMirror() error { + if c.runAsMirror() { + if !c.AllowSingleProvider && !c.atLeastNMirrors(2) { + return errors.New("at least 2 providers need to be mirrored") + } else if c.AllowSingleProvider && !c.atLeastNMirrors(1) { + return errors.New("at least one provider must be mirrored") + } + } else if !c.AllowSingleProvider && c.atLeastNMirrors(1) { + return errors.New("found mirrors in a lister aggregator") + } + + return nil +} + func (c *config) setDefaults() { if c.Folder == "" { c.Folder = defaultFolder @@ -219,7 +254,11 @@ func (c *config) check() error { return err } - return c.checkProviders() + if err := c.checkProviders(); err != nil { + return err + } + + return c.checkMirror() } func loadConfig(path string) (*config, error) { diff --git a/cmd/csaf_aggregator/full.go b/cmd/csaf_aggregator/full.go index d4835ae..fef169c 100644 --- a/cmd/csaf_aggregator/full.go +++ b/cmd/csaf_aggregator/full.go @@ -24,6 +24,7 @@ import ( type fullJob struct { provider *provider aggregatorProvider *csaf.AggregatorCSAFProvider + work fullWorkFunc err error } @@ -61,11 +62,7 @@ func (w *worker) setupProviderFull(provider *provider) error { type fullWorkFunc func(*worker) (*csaf.AggregatorCSAFProvider, error) // fullWork handles the treatment of providers concurrently. -func (w *worker) fullWork( - wg *sync.WaitGroup, - doWork fullWorkFunc, - jobs <-chan *fullJob, -) { +func (w *worker) fullWork(wg *sync.WaitGroup, jobs <-chan *fullJob) { defer wg.Done() for j := range jobs { @@ -73,16 +70,15 @@ func (w *worker) fullWork( j.err = err continue } - j.aggregatorProvider, j.err = doWork(w) + j.aggregatorProvider, j.err = j.work(w) } } // full performs the complete lister/download func (p *processor) full() error { - var doWork fullWorkFunc - if p.cfg.runAsMirror() { + log.Println("Running in aggregator mode") // check if we need to setup a remote validator if p.cfg.RemoteValidatorOptions != nil { @@ -98,11 +94,7 @@ func (p *processor) full() error { p.remoteValidator = nil }() } - - doWork = (*worker).mirror - log.Println("Running in aggregator mode") } else { - doWork = (*worker).lister log.Println("Running in lister mode") } @@ -113,13 +105,22 @@ func (p *processor) full() error { for i := 1; i <= p.cfg.Workers; i++ { wg.Add(1) w := newWorker(i, p) - go w.fullWork(&wg, doWork, queue) + go w.fullWork(&wg, queue) } jobs := make([]fullJob, len(p.cfg.Providers)) - for i, p := range p.cfg.Providers { - jobs[i] = fullJob{provider: p} + for i, provider := range p.cfg.Providers { + var work fullWorkFunc + if provider.runAsMirror(p.cfg) { + work = (*worker).mirror + } else { + work = (*worker).lister + } + jobs[i] = fullJob{ + provider: provider, + work: work, + } queue <- &jobs[i] } close(queue) diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index 6936c59..3eeb338 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -101,8 +101,14 @@ domain rate insecure write_indices +category ``` +If you want an entry to be listed instead of mirrored +in a `aggregator.category == "aggregator"` instance, +set `category` to `lister` in the entry. +Otherwise it is recommended to not set `category` for entries. + #### Example config file @@ -136,6 +142,7 @@ insecure = true create_service_document = true # rate = 1.5 # insecure = true + category = "lister" [[providers]] name = "local-dev-provider2" @@ -143,5 +150,14 @@ insecure = true # rate = 1.2 # insecure = true write_indices = true + category = "aggregator" + +[[providers]] + name = "local-dev-provider3" + domain = "localhost" +# rate = 1.8 +# insecure = true + write_indices = true + category = "aggregator" ``` diff --git a/docs/examples/aggregator.toml b/docs/examples/aggregator.toml index 13771b9..0ca585d 100644 --- a/docs/examples/aggregator.toml +++ b/docs/examples/aggregator.toml @@ -14,6 +14,8 @@ insecure = true # allow_single_provider = true [aggregator] + # Set if this instance shall be a mirror (aka `aggregator`) or a `lister`. + # This determines the default value for the entries in [[provider]]. category = "aggregator" name = "Example Development CSAF Aggregator" contact_details = "some @ somewhere" @@ -34,3 +36,13 @@ insecure = true # rate = 1.2 # insecure = true write_indices = true + +[[providers]] + name = "local-dev-provider3" + domain = "localhost" +# rate = 1.8 +# insecure = true + write_indices = true + # If aggregator.category == "aggreator", set for an entry that should + # be listed in addition: + category = "lister"