Allow configuration of time range and ignore patterns

cmd/csaf_downloader/config.go

@@ -126,7 +126,7 @@ func parseArgsConfig() ([]string, *config, error) {
 }
 
 // prepareDirectory ensures that the working directory
-// exists and is setup properly.
+// exists and is set up properly.
 func (cfg *config) prepareDirectory() error {
 	// If not given use current working directory.
 	if cfg.Directory == "" {
@@ -213,7 +213,7 @@ func (cfg *config) prepareCertificates() error {
 	return nil
 }
 
-// Prepare prepares internal state of a loaded configuration.
+// GetDownloadConfig Prepare prepares internal state of a loaded configuration.
 func (cfg *config) GetDownloadConfig() (*downloader.Config, error) {
 	for _, prepare := range []func(*config) error{
 		(*config).prepareDirectory,
@@ -233,7 +233,7 @@ func (cfg *config) GetDownloadConfig() (*downloader.Config, error) {
 		ClientPassphrase:     cfg.ClientPassphrase,
 		Rate:                 cfg.Rate,
 		Worker:               cfg.Worker,
-		Range:                cfg.Range,
+		Range:                (*[2]time.Time)(cfg.Range),
 		IgnorePattern:        cfg.ignorePattern,
 		ExtraHeader:          cfg.ExtraHeader,
 

lib/downloader/config.go

@@ -13,9 +13,8 @@ import (
 	"fmt"
 	"log/slog"
 	"net/http"
-
-	"github.com/csaf-poc/csaf_distribution/v3/internal/filter"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/models"
+	"regexp"
+	"time"
 )
 
 // ValidationMode specifies the strict the validation is.
@@ -37,8 +36,8 @@ type Config struct {
 	ClientPassphrase     *string
 	Rate                 *float64
 	Worker               int
-	Range                *models.TimeRange
-	IgnorePattern        filter.PatternMatcher
+	Range                *[2]time.Time
+	IgnorePattern        []*regexp.Regexp
 	ExtraHeader          http.Header
 
 	RemoteValidator string
@@ -82,7 +81,13 @@ func (vm *ValidationMode) UnmarshalFlag(value string) error {
 
 // ignoreFile returns true if the given URL should not be downloaded.
 func (cfg *Config) ignoreURL(u string) bool {
-	return cfg.IgnorePattern.Matches(u)
+	for _, expr := range cfg.IgnorePattern {
+		if expr.MatchString(u) {
+			return true
+		}
+	}
+	return false
+
 }
 
 // verbose is considered a log level equal or less debug.

lib/downloader/downloader.go

@@ -17,6 +17,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/csaf-poc/csaf_distribution/v3/internal/models"
 	"hash"
 	"io"
 	"log/slog"
@@ -25,7 +26,6 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 	"golang.org/x/time/rate"
@@ -56,11 +56,6 @@ type DownloadedDocument struct {
 	ValStatus ValidationStatus
 }
 
-// failedValidationDir is the name of the sub folder
-// where advisories are stored that fail validation in
-// unsafe mode.
-const failedValidationDir = "failed_validation"
-
 // NewDownloader constructs a new downloader given the configuration.
 func NewDownloader(cfg *Config) (*Downloader, error) {
 	var validator csaf.RemoteValidator
@@ -179,7 +174,7 @@ func (d *Downloader) enumerate(domain string) error {
 	loader := csaf.NewProviderMetadataLoader(client)
 	lpmd := loader.Enumerate(domain)
 
-	docs := []any{}
+	var docs []any
 
 	for _, pmd := range lpmd {
 		if d.cfg.verbose() {
@@ -249,9 +244,10 @@ func (d *Downloader) download(ctx context.Context, domain string) error {
 
 	// Do we need time range based filtering?
 	if d.cfg.Range != nil {
+		timeRange := models.NewTimeInterval(d.cfg.Range[0], d.cfg.Range[1])
 		d.cfg.Logger.Debug("Setting up filter to accept advisories within",
-			"timerange", d.cfg.Range)
-		afp.AgeAccept = d.cfg.Range.Contains
+			"timerange", timeRange)
+		afp.AgeAccept = timeRange.Contains
 	}
 
 	return afp.Process(func(label csaf.TLPLabel, files []csaf.AdvisoryFile) error {
@@ -428,7 +424,7 @@ func (d *Downloader) downloadWorker(
 		client = d.httpClient()
 		data   bytes.Buffer
 		stats  = stats{}
-		expr               = util.NewPathEval()
+		expr   = util.NewPathEval()
 	)
 
 	// Add collected stats back to total.