SystemSh0cker/gocsaf
1
0
Fork 0
mirror of https://github.com/gocsaf/csaf.git synced 2025-12-22 18:15:42 +01:00
Code Issues Releases Activity Actions

Move downloader to lib/downloader

Browse source
This commit is contained in:
koplas 2024-06-18 14:16:00 +02:00 • committed by koplas
parent 257c316894
commit fae4fdeabe
No known key found for this signature in database
7 changed files with 182 additions and 174 deletions
Download patch file Download diff file Expand all files Collapse all files

82
cmd/csaf_downloader/main.go
View file

@ -11,6 +11,7 @@ package main
import ( import (
"context" "context"
"github.com/csaf-poc/csaf_distribution/v3/lib/downloader"
"log/slog" "log/slog"
"os" "os"
"os/signal" "os/signal"
@ -18,12 +19,71 @@ import (
"github.com/csaf-poc/csaf_distribution/v3/internal/options" "github.com/csaf-poc/csaf_distribution/v3/internal/options"
) )
func run(cfg *config, domains []string) error { const (
d, err := newDownloader(cfg) defaultWorker = 2
defaultPreset = "mandatory"
defaultForwardQueue = 5
defaultValidationMode = downloader.ValidationStrict
defaultLogFile = "downloader.log"
defaultLogLevel = slog.LevelInfo
)
// configPaths are the potential file locations of the Config file.
var configPaths = []string{
"~/.config/csaf/downloader.toml",
"~/.csaf_downloader.toml",
"csaf_downloader.toml",
}
// parseArgsConfig parses the command line and if needed a config file.
func parseArgsConfig() ([]string, *downloader.Config, error) {
var (
logFile = defaultLogFile
logLevel = &options.LogLevel{Level: defaultLogLevel}
)
p := options.Parser[downloader.Config]{
DefaultConfigLocations: configPaths,
ConfigLocation: func(cfg *downloader.Config) string { return cfg.Config },
Usage: "[OPTIONS] domain...",
HasVersion: func(cfg *downloader.Config) bool { return cfg.Version },
SetDefaults: func(cfg *downloader.Config) {
cfg.Worker = defaultWorker
cfg.RemoteValidatorPresets = []string{defaultPreset}
cfg.ValidationMode = defaultValidationMode
cfg.ForwardQueue = defaultForwardQueue
cfg.LogFile = &logFile
cfg.LogLevel = logLevel
},
// Re-establish default values if not set.
EnsureDefaults: func(cfg *downloader.Config) {
if cfg.Worker == 0 {
cfg.Worker = defaultWorker
}
if cfg.RemoteValidatorPresets == nil {
cfg.RemoteValidatorPresets = []string{defaultPreset}
}
switch cfg.ValidationMode {
case downloader.ValidationStrict, downloader.ValidationUnsafe:
default:
cfg.ValidationMode = downloader.ValidationStrict
}
if cfg.LogFile == nil {
cfg.LogFile = &logFile
}
if cfg.LogLevel == nil {
cfg.LogLevel = logLevel
}
},
}
return p.Parse()
}
func run(cfg *downloader.Config, domains []string) error {
d, err := downloader.NewDownloader(cfg)
if err != nil { if err != nil {
return err return err
} }
defer d.close() defer d.Close()
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
@ -31,28 +91,28 @@ func run(cfg *config, domains []string) error {
defer stop() defer stop()
if cfg.ForwardURL != "" { if cfg.ForwardURL != "" {
f := newForwarder(cfg) f := downloader.NewForwarder(cfg)
go f.run() go f.Run()
defer func() { defer func() {
f.log() f.Log()
f.close() f.Close()
}() }()
d.forwarder = f d.Forwarder = f
} }
// If the enumerate-only flag is set, enumerate found PMDs, // If the enumerate-only flag is set, enumerate found PMDs,
// else use the normal load method // else use the normal load method
if cfg.EnumeratePMDOnly { if cfg.EnumeratePMDOnly {
return d.runEnumerate(domains) return d.RunEnumerate(domains)
} }
return d.run(ctx, domains) return d.Run(ctx, domains)
} }
func main() { func main() {
domains, cfg, err := parseArgsConfig() domains, cfg, err := parseArgsConfig()
options.ErrorCheck(err) options.ErrorCheck(err)
options.ErrorCheck(cfg.prepare()) options.ErrorCheck(cfg.Prepare())
if len(domains) == 0 { if len(domains) == 0 {
slog.Warn("No domains given.") slog.Warn("No domains given.")

111
cmd/csaf_downloader/config.go → lib/downloader/config.go
View file

@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de> // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2022 Intevation GmbH <https://intevation.de> // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
package main package downloader
import ( import (
"crypto/tls" "crypto/tls"
@ -25,23 +25,18 @@ import (
"github.com/csaf-poc/csaf_distribution/v3/internal/options" "github.com/csaf-poc/csaf_distribution/v3/internal/options"
) )
const ( // ValidationMode specifies the strict the validation is.
defaultWorker = 2 type ValidationMode string
defaultPreset = "mandatory"
defaultForwardQueue = 5
defaultValidationMode = validationStrict
defaultLogFile = "downloader.log"
defaultLogLevel = slog.LevelInfo
)
type validationMode string
const ( const (
validationStrict = validationMode("strict") // ValidationStrict skips advisories with failed validation.
validationUnsafe = validationMode("unsafe") ValidationStrict = ValidationMode("strict")
// ValidationUnsafe allows advisories with failed validation.
ValidationUnsafe = ValidationMode("unsafe")
) )
type config struct { // Config provides the download configuration.
type Config struct {
Directory string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"` Directory string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"` Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"`
IgnoreSignatureCheck bool `long:"ignore_sigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignore_sigcheck"` IgnoreSignatureCheck bool `long:"ignore_sigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignore_sigcheck"`
@ -64,7 +59,7 @@ type config struct {
RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"` RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"`
//lint:ignore SA5008 We are using choice twice: strict, unsafe. //lint:ignore SA5008 We are using choice twice: strict, unsafe.
ValidationMode validationMode `long:"validation_mode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"` ValidationMode ValidationMode `long:"validation_mode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"`
ForwardURL string `long:"forward_url" description:"URL of HTTP endpoint to forward downloads to" value-name:"URL" toml:"forward_url"` ForwardURL string `long:"forward_url" description:"URL of HTTP endpoint to forward downloads to" value-name:"URL" toml:"forward_url"`
ForwardHeader http.Header `long:"forward_header" description:"One or more extra HTTP header fields used by forwarding" toml:"forward_header"` ForwardHeader http.Header `long:"forward_header" description:"One or more extra HTTP header fields used by forwarding" toml:"forward_header"`
@ -81,60 +76,10 @@ type config struct {
ignorePattern filter.PatternMatcher ignorePattern filter.PatternMatcher
} }
// configPaths are the potential file locations of the config file.
var configPaths = []string{
"~/.config/csaf/downloader.toml",
"~/.csaf_downloader.toml",
"csaf_downloader.toml",
}
// parseArgsConfig parses the command line and if need a config file.
func parseArgsConfig() ([]string, *config, error) {
var (
logFile = defaultLogFile
logLevel = &options.LogLevel{Level: defaultLogLevel}
)
p := options.Parser[config]{
DefaultConfigLocations: configPaths,
ConfigLocation: func(cfg *config) string { return cfg.Config },
Usage: "[OPTIONS] domain...",
HasVersion: func(cfg *config) bool { return cfg.Version },
SetDefaults: func(cfg *config) {
cfg.Worker = defaultWorker
cfg.RemoteValidatorPresets = []string{defaultPreset}
cfg.ValidationMode = defaultValidationMode
cfg.ForwardQueue = defaultForwardQueue
cfg.LogFile = &logFile
cfg.LogLevel = logLevel
},
// Re-establish default values if not set.
EnsureDefaults: func(cfg *config) {
if cfg.Worker == 0 {
cfg.Worker = defaultWorker
}
if cfg.RemoteValidatorPresets == nil {
cfg.RemoteValidatorPresets = []string{defaultPreset}
}
switch cfg.ValidationMode {
case validationStrict, validationUnsafe:
default:
cfg.ValidationMode = validationStrict
}
if cfg.LogFile == nil {
cfg.LogFile = &logFile
}
if cfg.LogLevel == nil {
cfg.LogLevel = logLevel
}
},
}
return p.Parse()
}
// UnmarshalText implements [encoding.TextUnmarshaler]. // UnmarshalText implements [encoding.TextUnmarshaler].
func (vm *validationMode) UnmarshalText(text []byte) error { func (vm *ValidationMode) UnmarshalText(text []byte) error {
switch m := validationMode(text); m { switch m := ValidationMode(text); m {
case validationStrict, validationUnsafe: case ValidationStrict, ValidationUnsafe:
*vm = m *vm = m
default: default:
return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe)"`, m) return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe)"`, m)
@ -143,8 +88,8 @@ func (vm *validationMode) UnmarshalText(text []byte) error {
} }
// UnmarshalFlag implements [flags.UnmarshalFlag]. // UnmarshalFlag implements [flags.UnmarshalFlag].
func (vm *validationMode) UnmarshalFlag(value string) error { func (vm *ValidationMode) UnmarshalFlag(value string) error {
var v validationMode var v ValidationMode
if err := v.UnmarshalText([]byte(value)); err != nil { if err := v.UnmarshalText([]byte(value)); err != nil {
return err return err
} }
@ -153,18 +98,18 @@ func (vm *validationMode) UnmarshalFlag(value string) error {
} }
// ignoreFile returns true if the given URL should not be downloaded. // ignoreFile returns true if the given URL should not be downloaded.
func (cfg *config) ignoreURL(u string) bool { func (cfg *Config) ignoreURL(u string) bool {
return cfg.ignorePattern.Matches(u) return cfg.ignorePattern.Matches(u)
} }
// verbose is considered a log level equal or less debug. // verbose is considered a log level equal or less debug.
func (cfg *config) verbose() bool { func (cfg *Config) verbose() bool {
return cfg.LogLevel.Level <= slog.LevelDebug return cfg.LogLevel.Level <= slog.LevelDebug
} }
// prepareDirectory ensures that the working directory // prepareDirectory ensures that the working directory
// exists and is setup properly. // exists and is setup properly.
func (cfg *config) prepareDirectory() error { func (cfg *Config) prepareDirectory() error {
// If not given use current working directory. // If not given use current working directory.
if cfg.Directory == "" { if cfg.Directory == "" {
dir, err := os.Getwd() dir, err := os.Getwd()
@ -198,7 +143,7 @@ func dropSubSeconds(_ []string, a slog.Attr) slog.Attr {
} }
// prepareLogging sets up the structured logging. // prepareLogging sets up the structured logging.
func (cfg *config) prepareLogging() error { func (cfg *Config) prepareLogging() error {
var w io.Writer var w io.Writer
if cfg.LogFile == nil || *cfg.LogFile == "" { if cfg.LogFile == nil || *cfg.LogFile == "" {
log.Println("using STDERR for logging") log.Println("using STDERR for logging")
@ -231,7 +176,7 @@ func (cfg *config) prepareLogging() error {
} }
// compileIgnorePatterns compiles the configure patterns to be ignored. // compileIgnorePatterns compiles the configure patterns to be ignored.
func (cfg *config) compileIgnorePatterns() error { func (cfg *Config) compileIgnorePatterns() error {
pm, err := filter.NewPatternMatcher(cfg.IgnorePattern) pm, err := filter.NewPatternMatcher(cfg.IgnorePattern)
if err != nil { if err != nil {
return err return err
@ -241,7 +186,7 @@ func (cfg *config) compileIgnorePatterns() error {
} }
// prepareCertificates loads the client side certificates used by the HTTP client. // prepareCertificates loads the client side certificates used by the HTTP client.
func (cfg *config) prepareCertificates() error { func (cfg *Config) prepareCertificates() error {
cert, err := certs.LoadCertificate( cert, err := certs.LoadCertificate(
cfg.ClientCert, cfg.ClientKey, cfg.ClientPassphrase) cfg.ClientCert, cfg.ClientKey, cfg.ClientPassphrase)
if err != nil { if err != nil {
@ -251,13 +196,13 @@ func (cfg *config) prepareCertificates() error {
return nil return nil
} }
// prepare prepares internal state of a loaded configuration. // Prepare prepares internal state of a loaded configuration.
func (cfg *config) prepare() error { func (cfg *Config) Prepare() error {
for _, prepare := range []func(*config) error{ for _, prepare := range []func(*Config) error{
(*config).prepareDirectory, (*Config).prepareDirectory,
(*config).prepareLogging, (*Config).prepareLogging,
(*config).prepareCertificates, (*Config).prepareCertificates,
(*config).compileIgnorePatterns, (*Config).compileIgnorePatterns,
} { } {
if err := prepare(cfg); err != nil { if err := prepare(cfg); err != nil {
return err return err

69
cmd/csaf_downloader/downloader.go → lib/downloader/downloader.go
View file

@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2022, 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de> // SPDX-FileCopyrightText: 2022, 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2022, 2023 Intevation GmbH <https://intevation.de> // Software-Engineering: 2022, 2023 Intevation GmbH <https://intevation.de>
package main package downloader
import ( import (
"bytes" "bytes"
@ -37,11 +37,12 @@ import (
"github.com/csaf-poc/csaf_distribution/v3/util" "github.com/csaf-poc/csaf_distribution/v3/util"
) )
type downloader struct { // Downloader provides the CSAF downloader.
cfg *config type Downloader struct {
cfg *Config
keys *crypto.KeyRing keys *crypto.KeyRing
validator csaf.RemoteValidator validator csaf.RemoteValidator
forwarder *forwarder Forwarder *Forwarder
mkdirMu sync.Mutex mkdirMu sync.Mutex
statsMu sync.Mutex statsMu sync.Mutex
stats stats stats stats
@ -52,7 +53,8 @@ type downloader struct {
// unsafe mode. // unsafe mode.
const failedValidationDir = "failed_validation" const failedValidationDir = "failed_validation"
func newDownloader(cfg *config) (*downloader, error) { // NewDownloader constructs a new downloader given the configuration.
func NewDownloader(cfg *Config) (*Downloader, error) {
var validator csaf.RemoteValidator var validator csaf.RemoteValidator
@ -70,13 +72,14 @@ func newDownloader(cfg *config) (*downloader, error) {
validator = csaf.SynchronizedRemoteValidator(validator) validator = csaf.SynchronizedRemoteValidator(validator)
} }
return &downloader{ return &Downloader{
cfg: cfg, cfg: cfg,
validator: validator, validator: validator,
}, nil }, nil
} }
func (d *downloader) close() { // Close closes the downloader.
func (d *Downloader) Close() {
if d.validator != nil { if d.validator != nil {
d.validator.Close() d.validator.Close()
d.validator = nil d.validator = nil
@ -84,7 +87,7 @@ func (d *downloader) close() {
} }
// addStats add stats to total stats // addStats add stats to total stats
func (d *downloader) addStats(o *stats) { func (d *Downloader) addStats(o *stats) {
d.statsMu.Lock() d.statsMu.Lock()
defer d.statsMu.Unlock() defer d.statsMu.Unlock()
d.stats.add(o) d.stats.add(o)
@ -102,7 +105,7 @@ func logRedirect(req *http.Request, via []*http.Request) error {
return nil return nil
} }
func (d *downloader) httpClient() util.Client { func (d *Downloader) httpClient() util.Client {
hClient := http.Client{} hClient := http.Client{}
@ -162,7 +165,7 @@ func httpLog(who string) func(string, string) {
} }
} }
func (d *downloader) enumerate(domain string) error { func (d *Downloader) enumerate(domain string) error {
client := d.httpClient() client := d.httpClient()
loader := csaf.NewProviderMetadataLoader(client) loader := csaf.NewProviderMetadataLoader(client)
@ -192,7 +195,7 @@ func (d *downloader) enumerate(domain string) error {
return nil return nil
} }
func (d *downloader) download(ctx context.Context, domain string) error { func (d *Downloader) download(ctx context.Context, domain string) error {
client := d.httpClient() client := d.httpClient()
loader := csaf.NewProviderMetadataLoader(client) loader := csaf.NewProviderMetadataLoader(client)
@ -248,7 +251,7 @@ func (d *downloader) download(ctx context.Context, domain string) error {
}) })
} }
func (d *downloader) downloadFiles( func (d *Downloader) downloadFiles(
ctx context.Context, ctx context.Context,
label csaf.TLPLabel, label csaf.TLPLabel,
files []csaf.AdvisoryFile, files []csaf.AdvisoryFile,
@ -297,7 +300,7 @@ allFiles:
return errors.Join(errs...) return errors.Join(errs...)
} }
func (d *downloader) loadOpenPGPKeys( func (d *Downloader) loadOpenPGPKeys(
client util.Client, client util.Client,
doc any, doc any,
base *url.URL, base *url.URL,
@ -389,7 +392,7 @@ func (d *downloader) loadOpenPGPKeys(
} }
// logValidationIssues logs the issues reported by the advisory schema validation. // logValidationIssues logs the issues reported by the advisory schema validation.
func (d *downloader) logValidationIssues(url string, errors []string, err error) { func (d *Downloader) logValidationIssues(url string, errors []string, err error) {
if err != nil { if err != nil {
slog.Error("Failed to validate", slog.Error("Failed to validate",
"url", url, "url", url,
@ -409,7 +412,7 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error)
} }
} }
func (d *downloader) downloadWorker( func (d *Downloader) downloadWorker(
ctx context.Context, ctx context.Context,
wg *sync.WaitGroup, wg *sync.WaitGroup,
label csaf.TLPLabel, label csaf.TLPLabel,
@ -584,9 +587,9 @@ nextAdvisory:
// Validate against CSAF schema. // Validate against CSAF schema.
schemaCheck := func() error { schemaCheck := func() error {
if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 { if errs, err := csaf.ValidateCSAF(doc); err != nil || len(errs) > 0 {
stats.schemaFailed++ stats.schemaFailed++
d.logValidationIssues(file.URL(), errors, err) d.logValidationIssues(file.URL(), errs, err)
return fmt.Errorf("schema validation for %q failed", file.URL()) return fmt.Errorf("schema validation for %q failed", file.URL())
} }
return nil return nil
@ -633,16 +636,16 @@ nextAdvisory:
if err := check(); err != nil { if err := check(); err != nil {
slog.Error("Validation check failed", "error", err) slog.Error("Validation check failed", "error", err)
valStatus.update(invalidValidationStatus) valStatus.update(invalidValidationStatus)
if d.cfg.ValidationMode == validationStrict { if d.cfg.ValidationMode == ValidationStrict {
continue nextAdvisory continue nextAdvisory
} }
} }
} }
valStatus.update(validValidationStatus) valStatus.update(validValidationStatus)
// Send to forwarder // Send to Forwarder
if d.forwarder != nil { if d.Forwarder != nil {
d.forwarder.forward( d.Forwarder.forward(
filename, data.String(), filename, data.String(),
valStatus, valStatus,
string(s256Data), string(s256Data),
@ -690,17 +693,17 @@ nextAdvisory:
} }
// Write advisory to file // Write advisory to file
path := filepath.Join(lastDir, filename) filePath := filepath.Join(lastDir, filename)
// Write data to disk. // Write data to disk.
for _, x := range []struct { for _, x := range []struct {
p string p string
d []byte d []byte
}{ }{
{path, data.Bytes()}, {filePath, data.Bytes()},
{path + ".sha256", s256Data}, {filePath + ".sha256", s256Data},
{path + ".sha512", s512Data}, {filePath + ".sha512", s512Data},
{path + ".asc", signData}, {filePath + ".asc", signData},
} { } {
if x.d != nil { if x.d != nil {
if err := os.WriteFile(x.p, x.d, 0644); err != nil { if err := os.WriteFile(x.p, x.d, 0644); err != nil {
@ -711,17 +714,17 @@ nextAdvisory:
} }
stats.succeeded++ stats.succeeded++
slog.Info("Written advisory", "path", path) slog.Info("Written advisory", "path", filePath)
} }
} }
func (d *downloader) mkdirAll(path string, perm os.FileMode) error { func (d *Downloader) mkdirAll(path string, perm os.FileMode) error {
d.mkdirMu.Lock() d.mkdirMu.Lock()
defer d.mkdirMu.Unlock() defer d.mkdirMu.Unlock()
return os.MkdirAll(path, perm) return os.MkdirAll(path, perm)
} }
func (d *downloader) checkSignature(data []byte, sign *crypto.PGPSignature) error { func (d *Downloader) checkSignature(data []byte, sign *crypto.PGPSignature) error {
pm := crypto.NewPlainMessage(data) pm := crypto.NewPlainMessage(data)
t := crypto.GetUnixTime() t := crypto.GetUnixTime()
return d.keys.VerifyDetached(pm, sign, t) return d.keys.VerifyDetached(pm, sign, t)
@ -767,8 +770,8 @@ func loadHash(client util.Client, p string) ([]byte, []byte, error) {
return hash, data.Bytes(), nil return hash, data.Bytes(), nil
} }
// run performs the downloads for all the given domains. // Run performs the downloads for all the given domains.
func (d *downloader) run(ctx context.Context, domains []string) error { func (d *Downloader) Run(ctx context.Context, domains []string) error {
defer d.stats.log() defer d.stats.log()
for _, domain := range domains { for _, domain := range domains {
if err := d.download(ctx, domain); err != nil { if err := d.download(ctx, domain); err != nil {
@ -778,8 +781,8 @@ func (d *downloader) run(ctx context.Context, domains []string) error {
return nil return nil
} }
// runEnumerate performs the enumeration of PMDs for all the given domains. // RunEnumerate performs the enumeration of PMDs for all the given domains.
func (d *downloader) runEnumerate(domains []string) error { func (d *Downloader) RunEnumerate(domains []string) error {
defer d.stats.log() defer d.stats.log()
for _, domain := range domains { for _, domain := range domains {
if err := d.enumerate(domain); err != nil { if err := d.enumerate(domain); err != nil {

52
cmd/csaf_downloader/forwarder.go → lib/downloader/forwarder.go
View file

@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de> // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2023 Intevation GmbH <https://intevation.de> // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
package main package downloader
import ( import (
"bytes" "bytes"
@ -44,46 +44,46 @@ func (vs *validationStatus) update(status validationStatus) {
} }
} }
// forwarder forwards downloaded advisories to a given // Forwarder forwards downloaded advisories to a given
// HTTP endpoint. // HTTP endpoint.
type forwarder struct { type Forwarder struct {
cfg *config cfg *Config
cmds chan func(*forwarder) cmds chan func(*Forwarder)
client util.Client client util.Client
failed int failed int
succeeded int succeeded int
} }
// newForwarder creates a new forwarder. // NewForwarder creates a new Forwarder.
func newForwarder(cfg *config) *forwarder { func NewForwarder(cfg *Config) *Forwarder {
queue := cfg.ForwardQueue queue := cfg.ForwardQueue
if queue < 1 { if queue < 1 {
queue = 1 queue = 1
} }
return &forwarder{ return &Forwarder{
cfg: cfg, cfg: cfg,
cmds: make(chan func(*forwarder), queue), cmds: make(chan func(*Forwarder), queue),
} }
} }
// run runs the forwarder. Meant to be used in a Go routine. // Run runs the Forwarder. Meant to be used in a Go routine.
func (f *forwarder) run() { func (f *Forwarder) Run() {
defer slog.Debug("forwarder done") defer slog.Debug("Forwarder done")
for cmd := range f.cmds { for cmd := range f.cmds {
cmd(f) cmd(f)
} }
} }
// close terminates the forwarder. // Close terminates the Forwarder.
func (f *forwarder) close() { func (f *Forwarder) Close() {
close(f.cmds) close(f.cmds)
} }
// log logs the current statistics. // Log logs the current statistics.
func (f *forwarder) log() { func (f *Forwarder) Log() {
f.cmds <- func(f *forwarder) { f.cmds <- func(f *Forwarder) {
slog.Info("Forward statistics", slog.Info("Forward statistics",
"succeeded", f.succeeded, "succeeded", f.succeeded,
"failed", f.failed) "failed", f.failed)
@ -92,7 +92,7 @@ func (f *forwarder) log() {
// httpClient returns a cached HTTP client used for uploading // httpClient returns a cached HTTP client used for uploading
// the advisories to the configured HTTP endpoint. // the advisories to the configured HTTP endpoint.
func (f *forwarder) httpClient() util.Client { func (f *Forwarder) httpClient() util.Client {
if f.client != nil { if f.client != nil {
return f.client return f.client
} }
@ -122,7 +122,7 @@ func (f *forwarder) httpClient() util.Client {
if f.cfg.verbose() { if f.cfg.verbose() {
client = &util.LoggingClient{ client = &util.LoggingClient{
Client: client, Client: client,
Log: httpLog("forwarder"), Log: httpLog("Forwarder"),
} }
} }
@ -137,7 +137,7 @@ func replaceExt(fname, nExt string) string {
} }
// buildRequest creates an HTTP request suited to forward the given advisory. // buildRequest creates an HTTP request suited to forward the given advisory.
func (f *forwarder) buildRequest( func (f *Forwarder) buildRequest(
filename, doc string, filename, doc string,
status validationStatus, status validationStatus,
sha256, sha512 string, sha256, sha512 string,
@ -189,7 +189,7 @@ func (f *forwarder) buildRequest(
// storeFailedAdvisory stores an advisory in a special folder // storeFailedAdvisory stores an advisory in a special folder
// in case the forwarding failed. // in case the forwarding failed.
func (f *forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) error { func (f *Forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) error {
// Create special folder if it does not exist. // Create special folder if it does not exist.
dir := filepath.Join(f.cfg.Directory, failedForwardDir) dir := filepath.Join(f.cfg.Directory, failedForwardDir)
if err := os.MkdirAll(dir, 0755); err != nil { if err := os.MkdirAll(dir, 0755); err != nil {
@ -215,7 +215,7 @@ func (f *forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) er
} }
// storeFailed is a logging wrapper around storeFailedAdvisory. // storeFailed is a logging wrapper around storeFailedAdvisory.
func (f *forwarder) storeFailed(filename, doc, sha256, sha512 string) { func (f *Forwarder) storeFailed(filename, doc, sha256, sha512 string) {
f.failed++ f.failed++
if err := f.storeFailedAdvisory(filename, doc, sha256, sha512); err != nil { if err := f.storeFailedAdvisory(filename, doc, sha256, sha512); err != nil {
slog.Error("Storing advisory failed forwarding failed", slog.Error("Storing advisory failed forwarding failed",
@ -237,15 +237,15 @@ func limitedString(r io.Reader, max int) (string, error) {
} }
// forward sends a given document with filename, status and // forward sends a given document with filename, status and
// checksums to the forwarder. This is async to the degree // checksums to the Forwarder. This is async to the degree
// till the configured queue size is filled. // till the configured queue size is filled.
func (f *forwarder) forward( func (f *Forwarder) forward(
filename, doc string, filename, doc string,
status validationStatus, status validationStatus,
sha256, sha512 string, sha256, sha512 string,
) { ) {
// Run this in the main loop of the forwarder. // Run this in the main loop of the Forwarder.
f.cmds <- func(f *forwarder) { f.cmds <- func(f *Forwarder) {
req, err := f.buildRequest(filename, doc, status, sha256, sha512) req, err := f.buildRequest(filename, doc, status, sha256, sha512)
if err != nil { if err != nil {
slog.Error("building forward Request failed", slog.Error("building forward Request failed",

38
cmd/csaf_downloader/forwarder_test.go → lib/downloader/forwarder_test.go
View file

@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de> // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2023 Intevation GmbH <https://intevation.de> // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
package main package downloader
import ( import (
"bufio" "bufio"
@ -53,18 +53,18 @@ func TestForwarderLogStats(t *testing.T) {
lg := slog.New(h) lg := slog.New(h)
slog.SetDefault(lg) slog.SetDefault(lg)
cfg := &config{} cfg := &Config{}
fw := newForwarder(cfg) fw := NewForwarder(cfg)
fw.failed = 11 fw.failed = 11
fw.succeeded = 13 fw.succeeded = 13
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
defer close(done) defer close(done)
fw.run() fw.Run()
}() }()
fw.log() fw.Log()
fw.close() fw.Close()
<-done <-done
type fwStats struct { type fwStats struct {
@ -95,14 +95,14 @@ func TestForwarderLogStats(t *testing.T) {
} }
func TestForwarderHTTPClient(t *testing.T) { func TestForwarderHTTPClient(t *testing.T) {
cfg := &config{ cfg := &Config{
ForwardInsecure: true, ForwardInsecure: true,
ForwardHeader: http.Header{ ForwardHeader: http.Header{
"User-Agent": []string{"curl/7.55.1"}, "User-Agent": []string{"curl/7.55.1"},
}, },
LogLevel: &options.LogLevel{Level: slog.LevelDebug}, LogLevel: &options.LogLevel{Level: slog.LevelDebug},
} }
fw := newForwarder(cfg) fw := NewForwarder(cfg)
if c1, c2 := fw.httpClient(), fw.httpClient(); c1 != c2 { if c1, c2 := fw.httpClient(), fw.httpClient(); c1 != c2 {
t.Fatal("expected to return same client twice") t.Fatal("expected to return same client twice")
} }
@ -124,10 +124,10 @@ func TestForwarderReplaceExtension(t *testing.T) {
func TestForwarderBuildRequest(t *testing.T) { func TestForwarderBuildRequest(t *testing.T) {
// Good case ... // Good case ...
cfg := &config{ cfg := &Config{
ForwardURL: "https://example.com", ForwardURL: "https://example.com",
} }
fw := newForwarder(cfg) fw := NewForwarder(cfg)
req, err := fw.buildRequest( req, err := fw.buildRequest(
"test.json", "{}", "test.json", "{}",
@ -248,8 +248,8 @@ func TestStoreFailedAdvisory(t *testing.T) {
} }
defer os.RemoveAll(dir) defer os.RemoveAll(dir)
cfg := &config{Directory: dir} cfg := &Config{Directory: dir}
fw := newForwarder(cfg) fw := NewForwarder(cfg)
badDir := filepath.Join(dir, failedForwardDir) badDir := filepath.Join(dir, failedForwardDir)
if err := os.WriteFile(badDir, []byte("test"), 0664); err != nil { if err := os.WriteFile(badDir, []byte("test"), 0664); err != nil {
@ -301,8 +301,8 @@ func TestStoredFailed(t *testing.T) {
lg := slog.New(h) lg := slog.New(h)
slog.SetDefault(lg) slog.SetDefault(lg)
cfg := &config{Directory: dir} cfg := &Config{Directory: dir}
fw := newForwarder(cfg) fw := NewForwarder(cfg)
// An empty filename should lead to an error. // An empty filename should lead to an error.
fw.storeFailed("", "{}", "256", "512") fw.storeFailed("", "{}", "256", "512")
@ -385,11 +385,11 @@ func TestForwarderForward(t *testing.T) {
lg := slog.New(h) lg := slog.New(h)
slog.SetDefault(lg) slog.SetDefault(lg)
cfg := &config{ cfg := &Config{
ForwardURL: "http://example.com", ForwardURL: "http://example.com",
Directory: dir, Directory: dir,
} }
fw := newForwarder(cfg) fw := NewForwarder(cfg)
// Use the fact that http client is cached. // Use the fact that http client is cached.
fw.client = &fakeClient{} fw.client = &fakeClient{}
@ -398,7 +398,7 @@ func TestForwarderForward(t *testing.T) {
go func() { go func() {
defer close(done) defer close(done)
fw.run() fw.Run()
}() }()
// Iterate through states of http client. // Iterate through states of http client.
@ -412,7 +412,7 @@ func TestForwarderForward(t *testing.T) {
// Make buildRequest fail. // Make buildRequest fail.
wait := make(chan struct{}) wait := make(chan struct{})
fw.cmds <- func(f *forwarder) { fw.cmds <- func(f *Forwarder) {
f.cfg.ForwardURL = "%" f.cfg.ForwardURL = "%"
close(wait) close(wait)
} }
@ -423,7 +423,7 @@ func TestForwarderForward(t *testing.T) {
"256", "256",
"512") "512")
fw.close() fw.Close()
<-done <-done
} }

2
cmd/csaf_downloader/stats.go → lib/downloader/stats.go
View file

@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de> // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2023 Intevation GmbH <https://intevation.de> // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
package main package downloader
import "log/slog" import "log/slog"

2
cmd/csaf_downloader/stats_test.go → lib/downloader/stats_test.go
View file

@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de> // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2023 Intevation GmbH <https://intevation.de> // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
package main package downloader
import ( import (
"bytes" "bytes"