Add support for config files in downloader. (#404)

* Add support for config files in downloader.

* Add no-ini for the version flag, too.

* Add config file options in doc to downloader.

cmd/csaf_downloader/config.go

@@ -0,0 +1,66 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
+
+package main
+
+import (
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/mitchellh/go-homedir"
+)
+
+const defaultWorker = 2
+
+type config struct {
+	Directory            *string  `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"`
+	Insecure             bool     `long:"insecure" description:"Do not check TLS certificates from provider"`
+	IgnoreSignatureCheck bool     `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch"`
+	Version              bool     `long:"version" description:"Display version of the binary" no-ini:"true"`
+	Verbose              bool     `long:"verbose" short:"v" description:"Verbose output"`
+	Rate                 *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"`
+	Worker               int      `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM"`
+
+	ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
+
+	RemoteValidator        string   `long:"validator" description:"URL to validate documents remotely" value-name:"URL"`
+	RemoteValidatorCache   string   `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"`
+	RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"`
+
+	Config *string `short:"c" long:"config" description:"Path to config ini file" value-name:"INI-FILE" no-ini:"true"`
+}
+
+// iniPaths are the potential file locations of the the config file.
+var iniPaths = []string{
+	"~/.config/csaf/downloader.ini",
+	"~/.csaf_downloader.ini",
+	"csaf_downloader.ini",
+}
+
+// findIniFile looks for a file in the pre-defined paths in "iniPaths".
+// The returned value will be the name of file if found, otherwise an empty string.
+func findIniFile() string {
+	for _, f := range iniPaths {
+		name, err := homedir.Expand(f)
+		if err != nil {
+			log.Printf("warn: %v\n", err)
+			continue
+		}
+		if _, err := os.Stat(name); err == nil {
+			return name
+		}
+	}
+	return ""
+}
+
+// prepare prepares internal state of a loaded configuration.
+func (cfg *config) prepare() error {
+	// TODO: Implement me!
+	return nil
+}

cmd/csaf_downloader/downloader.go

@@ -37,7 +37,7 @@ import (
 )
 
 type downloader struct {
-	opts      *options
+	cfg       *config
 	directory string
 	keys      *crypto.KeyRing
 	eval      *util.PathEval
@@ -45,15 +45,15 @@ type downloader struct {
 	mkdirMu   sync.Mutex
 }
 
-func newDownloader(opts *options) (*downloader, error) {
+func newDownloader(cfg *config) (*downloader, error) {
 
 	var validator csaf.RemoteValidator
 
-	if opts.RemoteValidator != "" {
+	if cfg.RemoteValidator != "" {
 		validatorOptions := csaf.RemoteValidatorOptions{
-			URL:     opts.RemoteValidator,
+			URL:     cfg.RemoteValidator,
-			Presets: opts.RemoteValidatorPresets,
+			Presets: cfg.RemoteValidatorPresets,
-			Cache:   opts.RemoteValidatorCache,
+			Cache:   cfg.RemoteValidatorCache,
 		}
 		var err error
 		if validator, err = validatorOptions.Open(); err != nil {
@@ -64,7 +64,7 @@ func newDownloader(opts *options) (*downloader, error) {
 	}
 
 	return &downloader{
-		opts:      opts,
+		cfg:       cfg,
 		eval:      util.NewPathEval(),
 		validator: validator,
 	}, nil
@@ -82,7 +82,7 @@ func (d *downloader) httpClient() util.Client {
 	hClient := http.Client{}
 
 	var tlsConfig tls.Config
-	if d.opts.Insecure {
+	if d.cfg.Insecure {
 		tlsConfig.InsecureSkipVerify = true
 		hClient.Transport = &http.Transport{
 			TLSClientConfig: &tlsConfig,
@@ -92,23 +92,23 @@ func (d *downloader) httpClient() util.Client {
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(d.opts.ExtraHeader) > 0 {
+	if len(d.cfg.ExtraHeader) > 0 {
 		client = &util.HeaderClient{
 			Client: client,
-			Header: d.opts.ExtraHeader,
+			Header: d.cfg.ExtraHeader,
 		}
 	}
 
 	// Add optional URL logging.
-	if d.opts.Verbose {
+	if d.cfg.Verbose {
 		client = &util.LoggingClient{Client: client}
 	}
 
 	// Add optional rate limiting.
-	if d.opts.Rate != nil {
+	if d.cfg.Rate != nil {
 		client = &util.LimitingClient{
 			Client:  client,
-			Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1),
+			Limiter: rate.NewLimiter(rate.Limit(*d.cfg.Rate), 1),
 		}
 	}
 
@@ -122,7 +122,7 @@ func (d *downloader) download(ctx context.Context, domain string) error {
 
 	lpmd := loader.Load(domain)
 
-	if d.opts.Verbose {
+	if d.cfg.Verbose {
 		for i := range lpmd.Messages {
 			log.Printf("Loading provider-metadata.json for %q: %s\n",
 				domain, lpmd.Messages[i].Message)
@@ -181,7 +181,7 @@ func (d *downloader) downloadFiles(
 	}()
 
 	var n int
-	if n = d.opts.Worker; n < 1 {
+	if n = d.cfg.Worker; n < 1 {
 		n = 1
 	}
 
@@ -289,7 +289,7 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error)
 		return
 	}
 	if len(errors) > 0 {
-		if d.opts.Verbose {
+		if d.cfg.Verbose {
 			log.Printf("CSAF file %s has validation errors: %s\n",
 				url, strings.Join(errors, ", "))
 		} else {
@@ -372,7 +372,7 @@ nextAdvisory:
 
 		// Only hash when we have a remote counter part we can compare it with.
 		if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
-			if d.opts.Verbose {
+			if d.cfg.Verbose {
 				log.Printf("WARN: cannot fetch %s: %v\n", file.SHA256URL(), err)
 			}
 		} else {
@@ -381,7 +381,7 @@ nextAdvisory:
 		}
 
 		if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
-			if d.opts.Verbose {
+			if d.cfg.Verbose {
 				log.Printf("WARN: cannot fetch %s: %v\n", file.SHA512URL(), err)
 			}
 		} else {
@@ -423,7 +423,7 @@ nextAdvisory:
 			var sign *crypto.PGPSignature
 			sign, signData, err = loadSignature(client, file.SignURL())
 			if err != nil {
-				if d.opts.Verbose {
+				if d.cfg.Verbose {
 					log.Printf("downloading signature '%s' failed: %v\n",
 						file.SignURL(), err)
 				}
@@ -431,7 +431,7 @@ nextAdvisory:
 			if sign != nil {
 				if err := d.checkSignature(data.Bytes(), sign); err != nil {
 					log.Printf("Cannot verify signature for %s: %v\n", file.URL(), err)
-					if !d.opts.IgnoreSignatureCheck {
+					if !d.cfg.IgnoreSignatureCheck {
 						continue
 					}
 				}
@@ -560,7 +560,7 @@ func loadHash(client util.Client, p string) ([]byte, []byte, error) {
 // exists and is setup properly.
 func (d *downloader) prepareDirectory() error {
 	// If no special given use current working directory.
-	if d.opts.Directory == nil {
+	if d.cfg.Directory == nil {
 		dir, err := os.Getwd()
 		if err != nil {
 			return err
@@ -569,17 +569,17 @@ func (d *downloader) prepareDirectory() error {
 		return nil
 	}
 	// Use given directory
-	if _, err := os.Stat(*d.opts.Directory); err != nil {
+	if _, err := os.Stat(*d.cfg.Directory); err != nil {
 		// If it does not exist create it.
 		if os.IsNotExist(err) {
-			if err = os.MkdirAll(*d.opts.Directory, 0755); err != nil {
+			if err = os.MkdirAll(*d.cfg.Directory, 0755); err != nil {
 				return err
 			}
 		} else {
 			return err
 		}
 	}
-	d.directory = *d.opts.Directory
+	d.directory = *d.cfg.Directory
 	return nil
 }
 

cmd/csaf_downloader/main.go

@@ -13,32 +13,14 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"net/http"
 	"os"
 	"os/signal"
 
 	"github.com/csaf-poc/csaf_distribution/v2/util"
 	"github.com/jessevdk/go-flags"
+	"github.com/mitchellh/go-homedir"
 )
 
-const defaultWorker = 2
-
-type options struct {
-	Directory            *string  `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"`
-	Insecure             bool     `long:"insecure" description:"Do not check TLS certificates from provider"`
-	IgnoreSignatureCheck bool     `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch"`
-	Version              bool     `long:"version" description:"Display version of the binary"`
-	Verbose              bool     `long:"verbose" short:"v" description:"Verbose output"`
-	Rate                 *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"`
-	Worker               int      `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM"`
-
-	ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
-
-	RemoteValidator        string   `long:"validator" description:"URL to validate documents remotely" value-name:"URL"`
-	RemoteValidatorCache   string   `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"`
-	RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"`
-}
-
 func errCheck(err error) {
 	if err != nil {
 		if flags.WroteHelp(err) {
@@ -48,8 +30,8 @@ func errCheck(err error) {
 	}
 }
 
-func run(opts *options, domains []string) error {
+func run(cfg *config, domains []string) error {
-	d, err := newDownloader(opts)
+	d, err := newDownloader(cfg)
 	if err != nil {
 		return err
 	}
@@ -65,24 +47,38 @@ func run(opts *options, domains []string) error {
 
 func main() {
 
-	opts := &options{
+	cfg := &config{
 		Worker: defaultWorker,
 	}
 
-	parser := flags.NewParser(opts, flags.Default)
+	parser := flags.NewParser(cfg, flags.Default)
 	parser.Usage = "[OPTIONS] domain..."
 	domains, err := parser.Parse()
 	errCheck(err)
 
-	if opts.Version {
+	if cfg.Version {
 		fmt.Println(util.SemVersion)
 		return
 	}
 
+	if cfg.Config != nil {
+		iniParser := flags.NewIniParser(parser)
+		iniParser.ParseAsDefaults = true
+		name, err := homedir.Expand(*cfg.Config)
+		errCheck(err)
+		errCheck(iniParser.ParseFile(name))
+	} else if iniFile := findIniFile(); iniFile != "" {
+		iniParser := flags.NewIniParser(parser)
+		iniParser.ParseAsDefaults = true
+		errCheck(iniParser.ParseFile(iniFile))
+	}
+
+	errCheck(cfg.prepare())
+
 	if len(domains) == 0 {
 		log.Println("No domains given.")
 		return
 	}
 
-	errCheck(run(opts, domains))
+	errCheck(run(cfg, domains))
 }

docs/csaf_downloader.md

@@ -18,6 +18,7 @@ Application Options:
       --validator=URL          URL to validate documents remotely
       --validatorcache=FILE    FILE to cache remote validations
       --validatorpreset=       One or more presets to validate remotely (default: mandatory)
+  -c, --config=INI-FILE        Path to config ini file
 
 Help Options:
   -h, --help                   Show this help message
@@ -31,3 +32,26 @@ Increasing the number of workers opens more connections to the web servers
 to download more advisories at once. This may improve the overall speed of the download.
 However, since this also increases the load on the servers, their administrators could
 have taken countermeasures to limit this.
+
+If no config file is explictly given the follwing places are searched for a config file:
+```
+~/.config/csaf/downloader.ini
+~/.csaf_downloader.ini
+csaf_downloader.ini
+```
+
+with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
+
+Supported options in config files:
+```
+directory         # not set by default
+insecure          = false
+ignoresigcheck    = false
+verbose           = false
+# rate            # set to unlimited
+worker            = 2
+# header          # not set by default
+# validator       # not set by default
+# validatorcache  # not set by default
+validatorpreset   = "mandatory"
+```