From de0599ebe364762afcdf9d81691d0f313d6ad32c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 26 Jul 2023 03:22:33 +0200 Subject: [PATCH] Add time interval filtering to downloader. --- cmd/csaf_downloader/config.go | 19 ++++++++++++------- cmd/csaf_downloader/downloader.go | 5 +++++ csaf/advisories.go | 19 ++++++++++++++----- internal/models/models.go | 5 +++++ 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 45afcdc..cba4be4 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -10,7 +10,9 @@ package main import ( "net/http" + "time" + "github.com/csaf-poc/csaf_distribution/v2/internal/models" "github.com/csaf-poc/csaf_distribution/v2/internal/options" ) @@ -20,13 +22,14 @@ const ( ) type config struct { - Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"` - Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"` - IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignoresigcheck"` - Version bool `long:"version" description:"Display version of the binary" toml:"-"` - Verbose bool `long:"verbose" short:"v" description:"Verbose output" toml:"verbose"` - Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)" toml:"rate"` - Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"` + Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"` + Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"` + IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignoresigcheck"` + Version bool `long:"version" description:"Display version of the binary" toml:"-"` + Verbose bool `long:"verbose" short:"v" description:"Verbose output" toml:"verbose"` + Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)" toml:"rate"` + Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"` + Range *models.TimeRange `long:"timerange" short:"t" description:"RANGE of time from which advisories to download" value-name:"RANGE" toml:"timerange"` ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"` @@ -35,6 +38,8 @@ type config struct { RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validatorpreset"` Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"` + + ageAccept func(time.Time) bool } // configPaths are the potential file locations of the config file. diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 41bd076..690aa78 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -153,6 +153,11 @@ func (d *downloader) download(ctx context.Context, domain string) error { base, nil) + // Do we need time range based filtering? + if d.cfg.Range != nil { + afp.AgeAccept = d.cfg.Range.Contains + } + return afp.Process(func(label csaf.TLPLabel, files []csaf.AdvisoryFile) error { return d.downloadFiles(ctx, label, files) }) diff --git a/csaf/advisories.go b/csaf/advisories.go index a371b65..1d404a8 100644 --- a/csaf/advisories.go +++ b/csaf/advisories.go @@ -14,6 +14,7 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/csaf-poc/csaf_distribution/v2/util" ) @@ -71,11 +72,12 @@ func (haf HashedAdvisoryFile) SignURL() string { return haf.name(3, ".asc") } // AdvisoryFileProcessor implements the extraction of // advisory file names from a given provider metadata. type AdvisoryFileProcessor struct { - client util.Client - expr *util.PathEval - doc any - base *url.URL - log func(format string, args ...any) + AgeAccept func(time.Time) bool + client util.Client + expr *util.PathEval + doc any + base *url.URL + log func(format string, args ...any) } // NewAdvisoryFileProcessor constructs an filename extractor @@ -287,6 +289,13 @@ func (afp *AdvisoryFileProcessor) processROLIE( rfeed.Entries(func(entry *Entry) { + // Filter if we have date checking. + if afp.AgeAccept != nil { + if pub := time.Time(entry.Published); !pub.IsZero() && !afp.AgeAccept(pub) { + return + } + } + var self, sha256, sha512, sign string for i := range entry.Link { diff --git a/internal/models/models.go b/internal/models/models.go index af31c80..37e1daa 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -79,3 +79,8 @@ func (tr *TimeRange) UnmarshalText(text []byte) error { *tr = NewTimeInterval(start, end) return nil } + +// Contains return true if the given time is inside this time interval. +func (tr TimeRange) Contains(t time.Time) bool { + return !(t.Before(tr[0]) || t.After(tr[1])) +}