SystemSh0cker/gocsaf
1
0
Fork 0
mirror of https://github.com/gocsaf/csaf.git synced 2025-12-22 11:55:40 +01:00
Code Issues Releases Activity Actions

Add support for config files in downloader. (#404)

Browse source 
* Add support for config files in downloader.

* Add no-ini for the version flag, too.

* Add config file options in doc to downloader.
This commit is contained in:
Sascha L. Teichmann 2023-07-19 10:49:17 +02:00 • committed by GitHub
parent de27a668d1
commit 8630e8bac2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 135 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

66
cmd/csaf_downloader/config.go Normal file
View file

@ -0,0 +1,66 @@
// This file is Free Software under the MIT License
// without warranty, see README.md and LICENSES/MIT.txt for details.
//
// SPDX-License-Identifier: MIT
//
// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
package main
import (
"log"
"net/http"
"os"
"github.com/mitchellh/go-homedir"
)
const defaultWorker = 2
type config struct {
Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"`
IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch"`
Version bool `long:"version" description:"Display version of the binary" no-ini:"true"`
Verbose bool `long:"verbose" short:"v" description:"Verbose output"`
Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"`
Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM"`
ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL"`
RemoteValidatorCache string `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"`
RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"`
Config *string `short:"c" long:"config" description:"Path to config ini file" value-name:"INI-FILE" no-ini:"true"`
}
// iniPaths are the potential file locations of the the config file.
var iniPaths = []string{
"~/.config/csaf/downloader.ini",
"~/.csaf_downloader.ini",
"csaf_downloader.ini",
}
// findIniFile looks for a file in the pre-defined paths in "iniPaths".
// The returned value will be the name of file if found, otherwise an empty string.
func findIniFile() string {
for _, f := range iniPaths {
name, err := homedir.Expand(f)
if err != nil {
log.Printf("warn: %v\n", err)
continue
}
if _, err := os.Stat(name); err == nil {
return name
}
}
return ""
}
// prepare prepares internal state of a loaded configuration.
func (cfg *config) prepare() error {
// TODO: Implement me!
return nil
}

48
cmd/csaf_downloader/downloader.go
View file

@ -37,7 +37,7 @@ import (
)
type downloader struct {
opts *options
cfg *config
directory string
keys *crypto.KeyRing
eval *util.PathEval
@ -45,15 +45,15 @@ type downloader struct {
mkdirMu sync.Mutex
}
func newDownloader(opts *options) (*downloader, error) {
func newDownloader(cfg *config) (*downloader, error) {
var validator csaf.RemoteValidator
if opts.RemoteValidator != "" {
if cfg.RemoteValidator != "" {
validatorOptions := csaf.RemoteValidatorOptions{
URL: opts.RemoteValidator,
Presets: opts.RemoteValidatorPresets,
Cache: opts.RemoteValidatorCache,
URL: cfg.RemoteValidator,
Presets: cfg.RemoteValidatorPresets,
Cache: cfg.RemoteValidatorCache,
}
var err error
if validator, err = validatorOptions.Open(); err != nil {
@ -64,7 +64,7 @@ func newDownloader(opts *options) (*downloader, error) {
}
return &downloader{
opts: opts,
cfg: cfg,
eval: util.NewPathEval(),
validator: validator,
}, nil
@ -82,7 +82,7 @@ func (d *downloader) httpClient() util.Client {
hClient := http.Client{}
var tlsConfig tls.Config
if d.opts.Insecure {
if d.cfg.Insecure {
tlsConfig.InsecureSkipVerify = true
hClient.Transport = &http.Transport{
TLSClientConfig: &tlsConfig,
@ -92,23 +92,23 @@ func (d *downloader) httpClient() util.Client {
client := util.Client(&hClient)
// Add extra headers.
if len(d.opts.ExtraHeader) > 0 {
if len(d.cfg.ExtraHeader) > 0 {
client = &util.HeaderClient{
Client: client,
Header: d.opts.ExtraHeader,
Header: d.cfg.ExtraHeader,
}
}
// Add optional URL logging.
if d.opts.Verbose {
if d.cfg.Verbose {
client = &util.LoggingClient{Client: client}
}
// Add optional rate limiting.
if d.opts.Rate != nil {
if d.cfg.Rate != nil {
client = &util.LimitingClient{
Client: client,
Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1),
Limiter: rate.NewLimiter(rate.Limit(*d.cfg.Rate), 1),
}
}
@ -122,7 +122,7 @@ func (d *downloader) download(ctx context.Context, domain string) error {
lpmd := loader.Load(domain)
if d.opts.Verbose {
if d.cfg.Verbose {
for i := range lpmd.Messages {
log.Printf("Loading provider-metadata.json for %q: %s\n",
domain, lpmd.Messages[i].Message)
@ -181,7 +181,7 @@ func (d *downloader) downloadFiles(
}()
var n int
if n = d.opts.Worker; n < 1 {
if n = d.cfg.Worker; n < 1 {
n = 1
}
@ -289,7 +289,7 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error)
return
}
if len(errors) > 0 {
if d.opts.Verbose {
if d.cfg.Verbose {
log.Printf("CSAF file %s has validation errors: %s\n",
url, strings.Join(errors, ", "))
} else {
@ -372,7 +372,7 @@ nextAdvisory:
// Only hash when we have a remote counter part we can compare it with.
if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
if d.opts.Verbose {
if d.cfg.Verbose {
log.Printf("WARN: cannot fetch %s: %v\n", file.SHA256URL(), err)
}
} else {
@ -381,7 +381,7 @@ nextAdvisory:
}
if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
if d.opts.Verbose {
if d.cfg.Verbose {
log.Printf("WARN: cannot fetch %s: %v\n", file.SHA512URL(), err)
}
} else {
@ -423,7 +423,7 @@ nextAdvisory:
var sign *crypto.PGPSignature
sign, signData, err = loadSignature(client, file.SignURL())
if err != nil {
if d.opts.Verbose {
if d.cfg.Verbose {
log.Printf("downloading signature '%s' failed: %v\n",
file.SignURL(), err)
}
@ -431,7 +431,7 @@ nextAdvisory:
if sign != nil {
if err := d.checkSignature(data.Bytes(), sign); err != nil {
log.Printf("Cannot verify signature for %s: %v\n", file.URL(), err)
if !d.opts.IgnoreSignatureCheck {
if !d.cfg.IgnoreSignatureCheck {
continue
}
}
@ -560,7 +560,7 @@ func loadHash(client util.Client, p string) ([]byte, []byte, error) {
// exists and is setup properly.
func (d *downloader) prepareDirectory() error {
// If no special given use current working directory.
if d.opts.Directory == nil {
if d.cfg.Directory == nil {
dir, err := os.Getwd()
if err != nil {
return err
@ -569,17 +569,17 @@ func (d *downloader) prepareDirectory() error {
return nil
}
// Use given directory
if _, err := os.Stat(*d.opts.Directory); err != nil {
if _, err := os.Stat(*d.cfg.Directory); err != nil {
// If it does not exist create it.
if os.IsNotExist(err) {
if err = os.MkdirAll(*d.opts.Directory, 0755); err != nil {
if err = os.MkdirAll(*d.cfg.Directory, 0755); err != nil {
return err
}
} else {
return err
}
}
d.directory = *d.opts.Directory
d.directory = *d.cfg.Directory
return nil
}

46
cmd/csaf_downloader/main.go
View file

@ -13,32 +13,14 @@ import (
"context"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/jessevdk/go-flags"
"github.com/mitchellh/go-homedir"
)
const defaultWorker = 2
type options struct {
Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"`
IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch"`
Version bool `long:"version" description:"Display version of the binary"`
Verbose bool `long:"verbose" short:"v" description:"Verbose output"`
Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"`
Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM"`
ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL"`
RemoteValidatorCache string `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"`
RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"`
}
func errCheck(err error) {
if err != nil {
if flags.WroteHelp(err) {
@ -48,8 +30,8 @@ func errCheck(err error) {
}
}
func run(opts *options, domains []string) error {
d, err := newDownloader(opts)
func run(cfg *config, domains []string) error {
d, err := newDownloader(cfg)
if err != nil {
return err
}
@ -65,24 +47,38 @@ func run(opts *options, domains []string) error {
func main() {
opts := &options{
cfg := &config{
Worker: defaultWorker,
}
parser := flags.NewParser(opts, flags.Default)
parser := flags.NewParser(cfg, flags.Default)
parser.Usage = "[OPTIONS] domain..."
domains, err := parser.Parse()
errCheck(err)
if opts.Version {
if cfg.Version {
fmt.Println(util.SemVersion)
return
}
if cfg.Config != nil {
iniParser := flags.NewIniParser(parser)
iniParser.ParseAsDefaults = true
name, err := homedir.Expand(*cfg.Config)
errCheck(err)
errCheck(iniParser.ParseFile(name))
} else if iniFile := findIniFile(); iniFile != "" {
iniParser := flags.NewIniParser(parser)
iniParser.ParseAsDefaults = true
errCheck(iniParser.ParseFile(iniFile))
}
errCheck(cfg.prepare())
if len(domains) == 0 {
log.Println("No domains given.")
return
}
errCheck(run(opts, domains))
errCheck(run(cfg, domains))
}