diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go new file mode 100644 index 0000000..61a5d0d --- /dev/null +++ b/cmd/csaf_downloader/config.go @@ -0,0 +1,66 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +// Software-Engineering: 2022 Intevation GmbH + +package main + +import ( + "log" + "net/http" + "os" + + "github.com/mitchellh/go-homedir" +) + +const defaultWorker = 2 + +type config struct { + Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"` + Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"` + IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch"` + Version bool `long:"version" description:"Display version of the binary" no-ini:"true"` + Verbose bool `long:"verbose" short:"v" description:"Verbose output"` + Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"` + Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM"` + + ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"` + + RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL"` + RemoteValidatorCache string `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"` + RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"` + + Config *string `short:"c" long:"config" description:"Path to config ini file" value-name:"INI-FILE" no-ini:"true"` +} + +// iniPaths are the potential file locations of the the config file. +var iniPaths = []string{ + "~/.config/csaf/downloader.ini", + "~/.csaf_downloader.ini", + "csaf_downloader.ini", +} + +// findIniFile looks for a file in the pre-defined paths in "iniPaths". +// The returned value will be the name of file if found, otherwise an empty string. +func findIniFile() string { + for _, f := range iniPaths { + name, err := homedir.Expand(f) + if err != nil { + log.Printf("warn: %v\n", err) + continue + } + if _, err := os.Stat(name); err == nil { + return name + } + } + return "" +} + +// prepare prepares internal state of a loaded configuration. +func (cfg *config) prepare() error { + // TODO: Implement me! + return nil +} diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 712705e..41bd076 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -37,7 +37,7 @@ import ( ) type downloader struct { - opts *options + cfg *config directory string keys *crypto.KeyRing eval *util.PathEval @@ -45,15 +45,15 @@ type downloader struct { mkdirMu sync.Mutex } -func newDownloader(opts *options) (*downloader, error) { +func newDownloader(cfg *config) (*downloader, error) { var validator csaf.RemoteValidator - if opts.RemoteValidator != "" { + if cfg.RemoteValidator != "" { validatorOptions := csaf.RemoteValidatorOptions{ - URL: opts.RemoteValidator, - Presets: opts.RemoteValidatorPresets, - Cache: opts.RemoteValidatorCache, + URL: cfg.RemoteValidator, + Presets: cfg.RemoteValidatorPresets, + Cache: cfg.RemoteValidatorCache, } var err error if validator, err = validatorOptions.Open(); err != nil { @@ -64,7 +64,7 @@ func newDownloader(opts *options) (*downloader, error) { } return &downloader{ - opts: opts, + cfg: cfg, eval: util.NewPathEval(), validator: validator, }, nil @@ -82,7 +82,7 @@ func (d *downloader) httpClient() util.Client { hClient := http.Client{} var tlsConfig tls.Config - if d.opts.Insecure { + if d.cfg.Insecure { tlsConfig.InsecureSkipVerify = true hClient.Transport = &http.Transport{ TLSClientConfig: &tlsConfig, @@ -92,23 +92,23 @@ func (d *downloader) httpClient() util.Client { client := util.Client(&hClient) // Add extra headers. - if len(d.opts.ExtraHeader) > 0 { + if len(d.cfg.ExtraHeader) > 0 { client = &util.HeaderClient{ Client: client, - Header: d.opts.ExtraHeader, + Header: d.cfg.ExtraHeader, } } // Add optional URL logging. - if d.opts.Verbose { + if d.cfg.Verbose { client = &util.LoggingClient{Client: client} } // Add optional rate limiting. - if d.opts.Rate != nil { + if d.cfg.Rate != nil { client = &util.LimitingClient{ Client: client, - Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1), + Limiter: rate.NewLimiter(rate.Limit(*d.cfg.Rate), 1), } } @@ -122,7 +122,7 @@ func (d *downloader) download(ctx context.Context, domain string) error { lpmd := loader.Load(domain) - if d.opts.Verbose { + if d.cfg.Verbose { for i := range lpmd.Messages { log.Printf("Loading provider-metadata.json for %q: %s\n", domain, lpmd.Messages[i].Message) @@ -181,7 +181,7 @@ func (d *downloader) downloadFiles( }() var n int - if n = d.opts.Worker; n < 1 { + if n = d.cfg.Worker; n < 1 { n = 1 } @@ -289,7 +289,7 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error) return } if len(errors) > 0 { - if d.opts.Verbose { + if d.cfg.Verbose { log.Printf("CSAF file %s has validation errors: %s\n", url, strings.Join(errors, ", ")) } else { @@ -372,7 +372,7 @@ nextAdvisory: // Only hash when we have a remote counter part we can compare it with. if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil { - if d.opts.Verbose { + if d.cfg.Verbose { log.Printf("WARN: cannot fetch %s: %v\n", file.SHA256URL(), err) } } else { @@ -381,7 +381,7 @@ nextAdvisory: } if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil { - if d.opts.Verbose { + if d.cfg.Verbose { log.Printf("WARN: cannot fetch %s: %v\n", file.SHA512URL(), err) } } else { @@ -423,7 +423,7 @@ nextAdvisory: var sign *crypto.PGPSignature sign, signData, err = loadSignature(client, file.SignURL()) if err != nil { - if d.opts.Verbose { + if d.cfg.Verbose { log.Printf("downloading signature '%s' failed: %v\n", file.SignURL(), err) } @@ -431,7 +431,7 @@ nextAdvisory: if sign != nil { if err := d.checkSignature(data.Bytes(), sign); err != nil { log.Printf("Cannot verify signature for %s: %v\n", file.URL(), err) - if !d.opts.IgnoreSignatureCheck { + if !d.cfg.IgnoreSignatureCheck { continue } } @@ -560,7 +560,7 @@ func loadHash(client util.Client, p string) ([]byte, []byte, error) { // exists and is setup properly. func (d *downloader) prepareDirectory() error { // If no special given use current working directory. - if d.opts.Directory == nil { + if d.cfg.Directory == nil { dir, err := os.Getwd() if err != nil { return err @@ -569,17 +569,17 @@ func (d *downloader) prepareDirectory() error { return nil } // Use given directory - if _, err := os.Stat(*d.opts.Directory); err != nil { + if _, err := os.Stat(*d.cfg.Directory); err != nil { // If it does not exist create it. if os.IsNotExist(err) { - if err = os.MkdirAll(*d.opts.Directory, 0755); err != nil { + if err = os.MkdirAll(*d.cfg.Directory, 0755); err != nil { return err } } else { return err } } - d.directory = *d.opts.Directory + d.directory = *d.cfg.Directory return nil } diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index 559b8af..cc3b6ef 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -13,32 +13,14 @@ import ( "context" "fmt" "log" - "net/http" "os" "os/signal" "github.com/csaf-poc/csaf_distribution/v2/util" "github.com/jessevdk/go-flags" + "github.com/mitchellh/go-homedir" ) -const defaultWorker = 2 - -type options struct { - Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"` - Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"` - IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch"` - Version bool `long:"version" description:"Display version of the binary"` - Verbose bool `long:"verbose" short:"v" description:"Verbose output"` - Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"` - Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM"` - - ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"` - - RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL"` - RemoteValidatorCache string `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"` - RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"` -} - func errCheck(err error) { if err != nil { if flags.WroteHelp(err) { @@ -48,8 +30,8 @@ func errCheck(err error) { } } -func run(opts *options, domains []string) error { - d, err := newDownloader(opts) +func run(cfg *config, domains []string) error { + d, err := newDownloader(cfg) if err != nil { return err } @@ -65,24 +47,38 @@ func run(opts *options, domains []string) error { func main() { - opts := &options{ + cfg := &config{ Worker: defaultWorker, } - parser := flags.NewParser(opts, flags.Default) + parser := flags.NewParser(cfg, flags.Default) parser.Usage = "[OPTIONS] domain..." domains, err := parser.Parse() errCheck(err) - if opts.Version { + if cfg.Version { fmt.Println(util.SemVersion) return } + if cfg.Config != nil { + iniParser := flags.NewIniParser(parser) + iniParser.ParseAsDefaults = true + name, err := homedir.Expand(*cfg.Config) + errCheck(err) + errCheck(iniParser.ParseFile(name)) + } else if iniFile := findIniFile(); iniFile != "" { + iniParser := flags.NewIniParser(parser) + iniParser.ParseAsDefaults = true + errCheck(iniParser.ParseFile(iniFile)) + } + + errCheck(cfg.prepare()) + if len(domains) == 0 { log.Println("No domains given.") return } - errCheck(run(opts, domains)) + errCheck(run(cfg, domains)) } diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index 91c4ddd..9540fb0 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -18,6 +18,7 @@ Application Options: --validator=URL URL to validate documents remotely --validatorcache=FILE FILE to cache remote validations --validatorpreset= One or more presets to validate remotely (default: mandatory) + -c, --config=INI-FILE Path to config ini file Help Options: -h, --help Show this help message @@ -31,3 +32,26 @@ Increasing the number of workers opens more connections to the web servers to download more advisories at once. This may improve the overall speed of the download. However, since this also increases the load on the servers, their administrators could have taken countermeasures to limit this. + +If no config file is explictly given the follwing places are searched for a config file: +``` +~/.config/csaf/downloader.ini +~/.csaf_downloader.ini +csaf_downloader.ini +``` + +with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems. + +Supported options in config files: +``` +directory # not set by default +insecure = false +ignoresigcheck = false +verbose = false +# rate # set to unlimited +worker = 2 +# header # not set by default +# validator # not set by default +# validatorcache # not set by default +validatorpreset = "mandatory" +```