diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 44df12f..cff9240 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.20.3 + go-version: 1.21.0 - name: Build run: go build -v ./cmd/... diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index e025ae6..eff11c2 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -9,7 +9,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.20.3 + go-version: 1.21.0 - name: Set up Node.js uses: actions/setup-node@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ba3c006..6b74934 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '^1.20.3' + go-version: '^1.21.0' - name: Build run: make dist diff --git a/README.md b/README.md index 4b94dd3..09aa341 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Download the binaries from the most recent release assets on Github. ### Build from sources -- A recent version of **Go** (1.20+) should be installed. [Go installation](https://go.dev/doc/install) +- A recent version of **Go** (1.21+) should be installed. [Go installation](https://go.dev/doc/install) - Clone the repository `git clone https://github.com/csaf-poc/csaf_distribution.git ` diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 723171b..3bfea97 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -10,6 +10,7 @@ package main import ( "crypto/tls" + "fmt" "net/http" "github.com/csaf-poc/csaf_distribution/v2/internal/certs" @@ -19,8 +20,17 @@ import ( ) const ( - defaultWorker = 2 - defaultPreset = "mandatory" + defaultWorker = 2 + defaultPreset = "mandatory" + defaultForwardQueue = 5 + defaultValidationMode = validationStrict +) + +type validationMode string + +const ( + validationStrict = validationMode("strict") + validationUnsafe = validationMode("unsafe") ) type config struct { @@ -32,6 +42,7 @@ type config struct { ClientPassphrase *string `long:"client-passphrase" description:"Optional passphrase for the client cert (limited, experimental, see doc)" value-name:"PASSPHRASE" toml:"client_passphrase"` Version bool `long:"version" description:"Display version of the binary" toml:"-"` Verbose bool `long:"verbose" short:"v" description:"Verbose output" toml:"verbose"` + NoStore bool `long:"nostore" short:"n" description:"Do not store files" toml:"no_store"` Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)" toml:"rate"` Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"` Range *models.TimeRange `long:"timerange" short:"t" description:"RANGE of time from which advisories to download" value-name:"RANGE" toml:"timerange"` @@ -43,6 +54,14 @@ type config struct { RemoteValidatorCache string `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validatorcache"` RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validatorpreset"` + //lint:ignore SA5008 We are using choice twice: strict, unsafe. + ValidationMode validationMode `long:"validationmode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"` + + ForwardURL string `long:"forwardurl" description:"URL of HTTP endpoint to forward downloads to" value-name:"URL" toml:"forward_url"` + ForwardHeader http.Header `long:"forwardheader" description:"One or more extra HTTP header fields used by forwarding" toml:"forward_header"` + ForwardQueue int `long:"forwardqueue" description:"Maximal queue LENGTH before forwarder" value-name:"LENGTH" toml:"forward_queue"` + ForwardInsecure bool `long:"forwardinsecure" description:"Do not check TLS certificates from forward endpoint" toml:"forward_insecure"` + Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"` clientCerts []tls.Certificate @@ -66,6 +85,8 @@ func parseArgsConfig() ([]string, *config, error) { SetDefaults: func(cfg *config) { cfg.Worker = defaultWorker cfg.RemoteValidatorPresets = []string{defaultPreset} + cfg.ValidationMode = defaultValidationMode + cfg.ForwardQueue = defaultForwardQueue }, // Re-establish default values if not set. EnsureDefaults: func(cfg *config) { @@ -75,11 +96,27 @@ func parseArgsConfig() ([]string, *config, error) { if cfg.RemoteValidatorPresets == nil { cfg.RemoteValidatorPresets = []string{defaultPreset} } + switch cfg.ValidationMode { + case validationStrict, validationUnsafe: + default: + cfg.ValidationMode = validationStrict + } }, } return p.Parse() } +// UnmarshalText implements [encoding/text.TextUnmarshaler]. +func (vm *validationMode) UnmarshalText(text []byte) error { + switch m := validationMode(text); m { + case validationStrict, validationUnsafe: + *vm = m + default: + return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe"`, m) + } + return nil +} + // ignoreFile returns true if the given URL should not be downloaded. func (cfg *config) ignoreURL(u string) bool { return cfg.ignorePattern.Matches(u) diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 9720aa5..68bf837 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -42,6 +42,7 @@ type downloader struct { keys *crypto.KeyRing eval *util.PathEval validator csaf.RemoteValidator + forwarder *forwarder mkdirMu sync.Mutex } @@ -424,18 +425,26 @@ nextAdvisory: } // Compare the checksums. - if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) { - log.Printf("SHA256 checksum of %s does not match.\n", file.URL()) - continue + s256Check := func() error { + if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) { + return fmt.Errorf("SHA256 checksum of %s does not match", file.URL()) + } + return nil } - if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) { - log.Printf("SHA512 checksum of %s does not match.\n", file.URL()) - continue + s512Check := func() error { + if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) { + return fmt.Errorf("SHA512 checksum of %s does not match", file.URL()) + } + return nil } - // Only check signature if we have loaded keys. - if d.keys != nil { + // Validate OpenPGG signature. + keysCheck := func() error { + // Only check signature if we have loaded keys. + if d.keys == nil { + return nil + } var sign *crypto.PGPSignature sign, signData, err = loadSignature(client, file.SignURL()) if err != nil { @@ -446,37 +455,82 @@ nextAdvisory: } if sign != nil { if err := d.checkSignature(data.Bytes(), sign); err != nil { - log.Printf("Cannot verify signature for %s: %v\n", file.URL(), err) if !d.cfg.IgnoreSignatureCheck { - continue + return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err) } } } + return nil } // Validate against CSAF schema. - if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 { - d.logValidationIssues(file.URL(), errors, err) - continue + schemaCheck := func() error { + if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 { + d.logValidationIssues(file.URL(), errors, err) + return fmt.Errorf("schema validation for %q failed", file.URL()) + } + return nil } - if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil { - log.Printf("Ignoring %s: %s.\n", file.URL(), err) - continue + // Validate if filename is conforming. + filenameCheck := func() error { + if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil { + return fmt.Errorf("filename not conforming %s: %s", file.URL(), err) + } + return nil } - // Validate against remote validator - if d.validator != nil { + // Validate against remote validator. + remoteValidatorCheck := func() error { + if d.validator == nil { + return nil + } rvr, err := d.validator.Validate(doc) if err != nil { errorCh <- fmt.Errorf( "calling remote validator on %q failed: %w", file.URL(), err) - continue + return nil } if !rvr.Valid { - log.Printf("Remote validation of %q failed\n", file.URL()) + return fmt.Errorf("remote validation of %q failed", file.URL()) } + return nil + } + + // Run all the validations. + valStatus := notValidatedValidationStatus + for _, check := range []func() error{ + s256Check, + s512Check, + keysCheck, + schemaCheck, + filenameCheck, + remoteValidatorCheck, + } { + if err := check(); err != nil { + // TODO: Improve logging. + log.Printf("check failed: %v\n", err) + valStatus.update(invalidValidationStatus) + if d.cfg.ValidationMode == validationStrict { + continue nextAdvisory + } + } + } + valStatus.update(validValidationStatus) + + // Send to forwarder + if d.forwarder != nil { + d.forwarder.forward( + filename, data.String(), + valStatus, + string(s256Data), + string(s512Data)) + } + + if d.cfg.NoStore { + // Do not write locally. + continue } if err := d.eval.Extract(`$.document.tracking.initial_release_date`, dateExtract, false, doc); err != nil { diff --git a/cmd/csaf_downloader/forwarder.go b/cmd/csaf_downloader/forwarder.go new file mode 100644 index 0000000..c84a131 --- /dev/null +++ b/cmd/csaf_downloader/forwarder.go @@ -0,0 +1,199 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +package main + +import ( + "bytes" + "crypto/tls" + "io" + "log" + "mime/multipart" + "net/http" + "path/filepath" + "strings" + + "github.com/csaf-poc/csaf_distribution/v2/internal/misc" + "github.com/csaf-poc/csaf_distribution/v2/util" +) + +// validationStatus represents the validation status +// known to the HTTP endpoint. +type validationStatus string + +const ( + validValidationStatus = validationStatus("valid") + invalidValidationStatus = validationStatus("invalid") + notValidatedValidationStatus = validationStatus("not_validated") +) + +func (vs *validationStatus) update(status validationStatus) { + // Cannot heal after it fails at least once. + if *vs != invalidValidationStatus { + *vs = status + } +} + +// forwarder forwards downloaded advisories to a given +// HTTP endpoint. +type forwarder struct { + cfg *config + cmds chan func(*forwarder) + client util.Client +} + +// newForwarder creates a new forwarder. +func newForwarder(cfg *config) *forwarder { + queue := max(1, cfg.ForwardQueue) + return &forwarder{ + cfg: cfg, + cmds: make(chan func(*forwarder), queue), + } +} + +// run runs the forwarder. Meant to be used in a Go routine. +func (f *forwarder) run() { + defer log.Println("debug: forwarder done") + + for cmd := range f.cmds { + cmd(f) + } +} + +// close terminates the forwarder. +func (f *forwarder) close() { + close(f.cmds) +} + +// httpClient returns a cached HTTP client used for uploading +// the advisories to the configured HTTP endpoint. +func (f *forwarder) httpClient() util.Client { + if f.client != nil { + return f.client + } + + hClient := http.Client{} + + var tlsConfig tls.Config + if f.cfg.ForwardInsecure { + tlsConfig.InsecureSkipVerify = true + } + + hClient.Transport = &http.Transport{ + TLSClientConfig: &tlsConfig, + } + + client := util.Client(&hClient) + + // Add extra headers. + if len(f.cfg.ForwardHeader) > 0 { + client = &util.HeaderClient{ + Client: client, + Header: f.cfg.ForwardHeader, + } + } + + // Add optional URL logging. + if f.cfg.Verbose { + client = &util.LoggingClient{Client: client} + } + + f.client = client + return f.client +} + +// replaceExt replaces the extension of a given filename. +func replaceExt(fname, nExt string) string { + ext := filepath.Ext(fname) + return fname[:len(fname)-len(ext)] + nExt +} + +// forward sends a given document with filename, status and +// checksums to the forwarder. This is async to the degree +// till the configured queue size is filled. +func (f *forwarder) forward( + filename, doc string, + status validationStatus, + sha256, sha512 string, +) { + buildRequest := func() (*http.Request, error) { + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + + var err error + part := func(name, fname, mimeType, content string) { + if err != nil { + return + } + if fname == "" { + err = writer.WriteField(name, content) + return + } + var w io.Writer + if w, err = misc.CreateFormFile(writer, name, fname, mimeType); err == nil { + _, err = w.Write([]byte(content)) + } + } + + base := filepath.Base(filename) + part("advisory", base, "application/json", doc) + part("validation_status", "", "text/plain", string(status)) + if sha256 != "" { + part("hash-256", replaceExt(base, ".sha256"), "text/plain", sha256) + } + if sha512 != "" { + part("hash-512", replaceExt(base, ".sha512"), "text/plain", sha512) + } + + if err != nil { + return nil, err + } + + if err := writer.Close(); err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, f.cfg.ForwardURL, body) + if err != nil { + return nil, err + } + contentType := writer.FormDataContentType() + req.Header.Set("Content-Type", contentType) + return req, nil + } + + // Run this in the main loop of the forwarder. + f.cmds <- func(f *forwarder) { + req, err := buildRequest() + if err != nil { + // TODO: improve logging + log.Printf("error: %v\n", err) + return + } + res, err := f.httpClient().Do(req) + if err != nil { + // TODO: improve logging + log.Printf("error: %v\n", err) + return + } + if res.StatusCode != http.StatusCreated { + // TODO: improve logging + defer res.Body.Close() + var msg strings.Builder + io.Copy(&msg, io.LimitReader(res.Body, 512)) + var dots string + if msg.Len() >= 512 { + dots = "..." + } + log.Printf("error: %s: %q (%d)\n", + filename, msg.String()+dots, res.StatusCode) + } else { + log.Printf("info: forwarding %q succeeded\n", filename) + } + } +} diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index 33d3b32..fa8b232 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -30,6 +30,13 @@ func run(cfg *config, domains []string) error { ctx, stop := signal.NotifyContext(ctx, os.Interrupt) defer stop() + if cfg.ForwardURL != "" { + f := newForwarder(cfg) + go f.run() + defer f.close() + d.forwarder = f + } + return d.run(ctx, domains) } diff --git a/cmd/csaf_uploader/processor.go b/cmd/csaf_uploader/processor.go index 11f6ed6..9fcc994 100644 --- a/cmd/csaf_uploader/processor.go +++ b/cmd/csaf_uploader/processor.go @@ -18,7 +18,6 @@ import ( "log" "mime/multipart" "net/http" - "net/textproto" "os" "path/filepath" "strings" @@ -28,6 +27,7 @@ import ( "github.com/ProtonMail/gopenpgp/v2/crypto" "github.com/csaf-poc/csaf_distribution/v2/csaf" + "github.com/csaf-poc/csaf_distribution/v2/internal/misc" "github.com/csaf-poc/csaf_distribution/v2/util" ) @@ -103,20 +103,6 @@ func (p *processor) create() error { return nil } -var escapeQuotes = strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace - -// createFormFile creates an [io.Writer] like [mime/multipart.Writer.CreateFromFile]. -// This version allows to set the mime type, too. -func createFormFile(w *multipart.Writer, fieldname, filename, mimeType string) (io.Writer, error) { - // Source: https://cs.opensource.google/go/go/+/refs/tags/go1.20:src/mime/multipart/writer.go;l=140 - h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", - fmt.Sprintf(`form-data; name="%s"; filename="%s"`, - escapeQuotes(fieldname), escapeQuotes(filename))) - h.Set("Content-Type", mimeType) - return w.CreatePart(h) -} - // uploadRequest creates the request for uploading a csaf document by passing the filename. // According to the flags values the multipart sections of the request are established. // It returns the created http request. @@ -151,7 +137,7 @@ func (p *processor) uploadRequest(filename string) (*http.Request, error) { // As the csaf_provider only accepts uploads with mime type // "application/json" we have to set this. - part, err := createFormFile( + part, err := misc.CreateFormFile( writer, "csaf", filepath.Base(filename), "application/json") if err != nil { return nil, err diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index 5be48fa..e858cff 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -7,28 +7,34 @@ A tool to download CSAF documents from CSAF providers. csaf_downloader [OPTIONS] domain... Application Options: - -d, --directory=DIR DIRectory to store the downloaded files in - --insecure Do not check TLS certificates from provider - --ignoresigcheck Ignore signature check results, just warn on mismatch - --client-cert=CERT-FILE TLS client certificate file (PEM encoded data) - --client-key=KEY-FILE TLS client private key file (PEM encoded data) - --client-passphrase=PASSPHRASE Optional passphrase for the client cert (limited, experimental, see doc) - --version Display version of the binary - -v, --verbose Verbose output - -r, --rate= The average upper limit of https operations per second (defaults to - unlimited) - -w, --worker=NUM NUMber of concurrent downloads (default: 2) - -t, --timerange=RANGE RANGE of time from which advisories to download - -f, --folder=FOLDER Download into a given subFOLDER - -i, --ignorepattern=PATTERN Do not download files if their URLs match any of the given PATTERNs - -H, --header= One or more extra HTTP header fields - --validator=URL URL to validate documents remotely - --validatorcache=FILE FILE to cache remote validations - --validatorpreset=PRESETS One or more PRESETS to validate remotely (default: [mandatory]) - -c, --config=TOML-FILE Path to config TOML file + -d, --directory=DIR DIRectory to store the downloaded files in + --insecure Do not check TLS certificates from provider + --ignoresigcheck Ignore signature check results, just warn on mismatch + --client-cert=CERT-FILE TLS client certificate file (PEM encoded data) + --client-key=KEY-FILE TLS client private key file (PEM encoded data) + --client-passphrase=PASSPHRASE Optional passphrase for the client cert (limited, experimental, see doc) + --version Display version of the binary + -v, --verbose Verbose output + -n, --nostore Do not store files + -r, --rate= The average upper limit of https operations per second (defaults to + unlimited) + -w, --worker=NUM NUMber of concurrent downloads (default: 2) + -t, --timerange=RANGE RANGE of time from which advisories to download + -f, --folder=FOLDER Download into a given subFOLDER + -i, --ignorepattern=PATTERN Do not download files if their URLs match any of the given PATTERNs + -H, --header= One or more extra HTTP header fields + --validator=URL URL to validate documents remotely + --validatorcache=FILE FILE to cache remote validations + --validatorpreset=PRESETS One or more PRESETS to validate remotely (default: [mandatory]) + -m, --validationmode=MODE[strict|unsafe] MODE how strict the validation is (default: strict) + --forwardurl=URL URL of HTTP endpoint to forward downloads to + --forwardheader= One or more extra HTTP header fields used by forwarding + --forwardqueue=LENGTH Maximal queue LENGTH before forwarder + --forwardinsecure Do not check TLS certificates from forward endpoint + -c, --config=TOML-FILE Path to config TOML file Help Options: - -h, --help Show this help message + -h, --help Show this help message ``` Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider. @@ -67,6 +73,11 @@ worker = 2 # validator # not set by default # validatorcache # not set by default validatorpreset = ["mandatory"] +validation_mode = "strict" +# forward_url # not set by default +# forward_header # not set by default +forward_queue = 5 +forward_insecure = false ``` The `timerange` parameter enables downloading advisories which last changes falls diff --git a/go.mod b/go.mod index 8cf34b0..dbbbcff 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/csaf-poc/csaf_distribution/v2 -go 1.20 +go 1.21 require ( github.com/BurntSushi/toml v1.3.2 diff --git a/internal/misc/doc.go b/internal/misc/doc.go new file mode 100644 index 0000000..1fec00a --- /dev/null +++ b/internal/misc/doc.go @@ -0,0 +1,10 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +// Package misc implements miscellaneous helper functions. +package misc diff --git a/internal/misc/mime.go b/internal/misc/mime.go new file mode 100644 index 0000000..0e699a3 --- /dev/null +++ b/internal/misc/mime.go @@ -0,0 +1,31 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +package misc + +import ( + "fmt" + "io" + "mime/multipart" + "net/textproto" + "strings" +) + +var escapeQuotes = strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace + +// CreateFormFile creates an [io.Writer] like [mime/multipart.Writer.CreateFromFile]. +// This version allows to set the mime type, too. +func CreateFormFile(w *multipart.Writer, fieldname, filename, mimeType string) (io.Writer, error) { + // Source: https://cs.opensource.google/go/go/+/refs/tags/go1.20:src/mime/multipart/writer.go;l=140 + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + escapeQuotes(fieldname), escapeQuotes(filename))) + h.Set("Content-Type", mimeType) + return w.CreatePart(h) +}