Downloader: Add forwarding to HTTP endpoint (#442)

* started with forwarding support in downloader

* Add missing files.

* Add missing files.

* Raise needed Go version

* More Go version bumping.

* Fix forwarding

* Go 1.21+ needed

* Make terminating forwarder more robust.

* Better var naming

* Remove dead code. Improve commentary.

* Prepare validation status adjustment.

* Move validations to functions to make them executable in a loop.

* Introduce validation mode flag (strict, unsafe)

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.3
+          go-version: 1.21.0
 
       - name: Build
         run: go build -v ./cmd/...

.github/workflows/itest.yml

@@ -9,7 +9,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.3
+          go-version: 1.21.0
 
       - name: Set up Node.js
         uses: actions/setup-node@v3

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-            go-version: '^1.20.3'
+            go-version: '^1.21.0'
 
       - name: Build
         run: make dist

README.md

@@ -37,7 +37,7 @@ Download the binaries from the most recent release assets on Github.
 
 ### Build from sources
 
-- A recent version of **Go** (1.20+) should be installed. [Go installation](https://go.dev/doc/install)
+- A recent version of **Go** (1.21+) should be installed. [Go installation](https://go.dev/doc/install)
 
 - Clone the repository `git clone https://github.com/csaf-poc/csaf_distribution.git `
 

cmd/csaf_downloader/config.go

@@ -10,6 +10,7 @@ package main
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net/http"
 
 	"github.com/csaf-poc/csaf_distribution/v2/internal/certs"
@@ -19,8 +20,17 @@ import (
 )
 
 const (
-	defaultWorker = 2
-	defaultPreset = "mandatory"
+	defaultWorker         = 2
+	defaultPreset         = "mandatory"
+	defaultForwardQueue   = 5
+	defaultValidationMode = validationStrict
+)
+
+type validationMode string
+
+const (
+	validationStrict = validationMode("strict")
+	validationUnsafe = validationMode("unsafe")
 )
 
 type config struct {
@@ -32,6 +42,7 @@ type config struct {
 	ClientPassphrase     *string           `long:"client-passphrase" description:"Optional passphrase for the client cert (limited, experimental, see doc)" value-name:"PASSPHRASE" toml:"client_passphrase"`
 	Version              bool              `long:"version" description:"Display version of the binary" toml:"-"`
 	Verbose              bool              `long:"verbose" short:"v" description:"Verbose output" toml:"verbose"`
+	NoStore              bool              `long:"nostore" short:"n" description:"Do not store files" toml:"no_store"`
 	Rate                 *float64          `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)" toml:"rate"`
 	Worker               int               `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"`
 	Range                *models.TimeRange `long:"timerange" short:"t" description:"RANGE of time from which advisories to download" value-name:"RANGE" toml:"timerange"`
@@ -43,6 +54,14 @@ type config struct {
 	RemoteValidatorCache   string   `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validatorcache"`
 	RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validatorpreset"`
 
+	//lint:ignore SA5008 We are using choice twice: strict, unsafe.
+	ValidationMode validationMode `long:"validationmode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"`
+
+	ForwardURL      string      `long:"forwardurl" description:"URL of HTTP endpoint to forward downloads to" value-name:"URL" toml:"forward_url"`
+	ForwardHeader   http.Header `long:"forwardheader" description:"One or more extra HTTP header fields used by forwarding" toml:"forward_header"`
+	ForwardQueue    int         `long:"forwardqueue" description:"Maximal queue LENGTH before forwarder" value-name:"LENGTH" toml:"forward_queue"`
+	ForwardInsecure bool        `long:"forwardinsecure" description:"Do not check TLS certificates from forward endpoint" toml:"forward_insecure"`
+
 	Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"`
 
 	clientCerts   []tls.Certificate
@@ -66,6 +85,8 @@ func parseArgsConfig() ([]string, *config, error) {
 		SetDefaults: func(cfg *config) {
 			cfg.Worker = defaultWorker
 			cfg.RemoteValidatorPresets = []string{defaultPreset}
+			cfg.ValidationMode = defaultValidationMode
+			cfg.ForwardQueue = defaultForwardQueue
 		},
 		// Re-establish default values if not set.
 		EnsureDefaults: func(cfg *config) {
@@ -75,11 +96,27 @@ func parseArgsConfig() ([]string, *config, error) {
 			if cfg.RemoteValidatorPresets == nil {
 				cfg.RemoteValidatorPresets = []string{defaultPreset}
 			}
+			switch cfg.ValidationMode {
+			case validationStrict, validationUnsafe:
+			default:
+				cfg.ValidationMode = validationStrict
+			}
 		},
 	}
 	return p.Parse()
 }
 
+// UnmarshalText implements [encoding/text.TextUnmarshaler].
+func (vm *validationMode) UnmarshalText(text []byte) error {
+	switch m := validationMode(text); m {
+	case validationStrict, validationUnsafe:
+		*vm = m
+	default:
+		return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe"`, m)
+	}
+	return nil
+}
+
 // ignoreFile returns true if the given URL should not be downloaded.
 func (cfg *config) ignoreURL(u string) bool {
 	return cfg.ignorePattern.Matches(u)

cmd/csaf_downloader/downloader.go

@@ -42,6 +42,7 @@ type downloader struct {
 	keys      *crypto.KeyRing
 	eval      *util.PathEval
 	validator csaf.RemoteValidator
+	forwarder *forwarder
 	mkdirMu   sync.Mutex
 }
 
@@ -424,18 +425,26 @@ nextAdvisory:
 		}
 
 		// Compare the checksums.
-		if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) {
-			log.Printf("SHA256 checksum of %s does not match.\n", file.URL())
-			continue
+		s256Check := func() error {
+			if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) {
+				return fmt.Errorf("SHA256 checksum of %s does not match", file.URL())
+			}
+			return nil
 		}
 
-		if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) {
-			log.Printf("SHA512 checksum of %s does not match.\n", file.URL())
-			continue
+		s512Check := func() error {
+			if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) {
+				return fmt.Errorf("SHA512 checksum of %s does not match", file.URL())
+			}
+			return nil
 		}
 
-		// Only check signature if we have loaded keys.
-		if d.keys != nil {
+		// Validate OpenPGG signature.
+		keysCheck := func() error {
+			// Only check signature if we have loaded keys.
+			if d.keys == nil {
+				return nil
+			}
 			var sign *crypto.PGPSignature
 			sign, signData, err = loadSignature(client, file.SignURL())
 			if err != nil {
@@ -446,37 +455,82 @@ nextAdvisory:
 			}
 			if sign != nil {
 				if err := d.checkSignature(data.Bytes(), sign); err != nil {
-					log.Printf("Cannot verify signature for %s: %v\n", file.URL(), err)
 					if !d.cfg.IgnoreSignatureCheck {
-						continue
+						return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err)
 					}
 				}
 			}
+			return nil
 		}
 
 		// Validate against CSAF schema.
-		if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 {
-			d.logValidationIssues(file.URL(), errors, err)
-			continue
+		schemaCheck := func() error {
+			if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 {
+				d.logValidationIssues(file.URL(), errors, err)
+				return fmt.Errorf("schema validation for %q failed", file.URL())
+			}
+			return nil
 		}
 
-		if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil {
-			log.Printf("Ignoring %s: %s.\n", file.URL(), err)
-			continue
+		// Validate if filename is conforming.
+		filenameCheck := func() error {
+			if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil {
+				return fmt.Errorf("filename not conforming %s: %s", file.URL(), err)
+			}
+			return nil
 		}
 
-		// Validate against remote validator
-		if d.validator != nil {
+		// Validate against remote validator.
+		remoteValidatorCheck := func() error {
+			if d.validator == nil {
+				return nil
+			}
 			rvr, err := d.validator.Validate(doc)
 			if err != nil {
 				errorCh <- fmt.Errorf(
 					"calling remote validator on %q failed: %w",
 					file.URL(), err)
-				continue
+				return nil
 			}
 			if !rvr.Valid {
-				log.Printf("Remote validation of %q failed\n", file.URL())
+				return fmt.Errorf("remote validation of %q failed", file.URL())
 			}
+			return nil
+		}
+
+		// Run all the validations.
+		valStatus := notValidatedValidationStatus
+		for _, check := range []func() error{
+			s256Check,
+			s512Check,
+			keysCheck,
+			schemaCheck,
+			filenameCheck,
+			remoteValidatorCheck,
+		} {
+			if err := check(); err != nil {
+				// TODO: Improve logging.
+				log.Printf("check failed: %v\n", err)
+				valStatus.update(invalidValidationStatus)
+				if d.cfg.ValidationMode == validationStrict {
+					continue nextAdvisory
+				}
+			}
+		}
+		valStatus.update(validValidationStatus)
+
+		// Send to forwarder
+		if d.forwarder != nil {
+			d.forwarder.forward(
+				filename, data.String(),
+				valStatus,
+				string(s256Data),
+				string(s512Data))
+		}
+
+		if d.cfg.NoStore {
+			// Do not write locally.
+			continue
 		}
 
 		if err := d.eval.Extract(`$.document.tracking.initial_release_date`, dateExtract, false, doc); err != nil {

cmd/csaf_downloader/forwarder.go

@@ -0,0 +1,199 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
+
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"log"
+	"mime/multipart"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/csaf-poc/csaf_distribution/v2/internal/misc"
+	"github.com/csaf-poc/csaf_distribution/v2/util"
+)
+
+// validationStatus represents the validation status
+// known to the HTTP endpoint.
+type validationStatus string
+
+const (
+	validValidationStatus        = validationStatus("valid")
+	invalidValidationStatus      = validationStatus("invalid")
+	notValidatedValidationStatus = validationStatus("not_validated")
+)
+
+func (vs *validationStatus) update(status validationStatus) {
+	// Cannot heal after it fails at least once.
+	if *vs != invalidValidationStatus {
+		*vs = status
+	}
+}
+
+// forwarder forwards downloaded advisories to a given
+// HTTP endpoint.
+type forwarder struct {
+	cfg    *config
+	cmds   chan func(*forwarder)
+	client util.Client
+}
+
+// newForwarder creates a new forwarder.
+func newForwarder(cfg *config) *forwarder {
+	queue := max(1, cfg.ForwardQueue)
+	return &forwarder{
+		cfg:  cfg,
+		cmds: make(chan func(*forwarder), queue),
+	}
+}
+
+// run runs the forwarder. Meant to be used in a Go routine.
+func (f *forwarder) run() {
+	defer log.Println("debug: forwarder done")
+
+	for cmd := range f.cmds {
+		cmd(f)
+	}
+}
+
+// close terminates the forwarder.
+func (f *forwarder) close() {
+	close(f.cmds)
+}
+
+// httpClient returns a cached HTTP client used for uploading
+// the advisories to the configured HTTP endpoint.
+func (f *forwarder) httpClient() util.Client {
+	if f.client != nil {
+		return f.client
+	}
+
+	hClient := http.Client{}
+
+	var tlsConfig tls.Config
+	if f.cfg.ForwardInsecure {
+		tlsConfig.InsecureSkipVerify = true
+	}
+
+	hClient.Transport = &http.Transport{
+		TLSClientConfig: &tlsConfig,
+	}
+
+	client := util.Client(&hClient)
+
+	// Add extra headers.
+	if len(f.cfg.ForwardHeader) > 0 {
+		client = &util.HeaderClient{
+			Client: client,
+			Header: f.cfg.ForwardHeader,
+		}
+	}
+
+	// Add optional URL logging.
+	if f.cfg.Verbose {
+		client = &util.LoggingClient{Client: client}
+	}
+
+	f.client = client
+	return f.client
+}
+
+// replaceExt replaces the extension of a given filename.
+func replaceExt(fname, nExt string) string {
+	ext := filepath.Ext(fname)
+	return fname[:len(fname)-len(ext)] + nExt
+}
+
+// forward sends a given document with filename, status and
+// checksums to the forwarder. This is async to the degree
+// till the configured queue size is filled.
+func (f *forwarder) forward(
+	filename, doc string,
+	status validationStatus,
+	sha256, sha512 string,
+) {
+	buildRequest := func() (*http.Request, error) {
+		body := new(bytes.Buffer)
+		writer := multipart.NewWriter(body)
+
+		var err error
+		part := func(name, fname, mimeType, content string) {
+			if err != nil {
+				return
+			}
+			if fname == "" {
+				err = writer.WriteField(name, content)
+				return
+			}
+			var w io.Writer
+			if w, err = misc.CreateFormFile(writer, name, fname, mimeType); err == nil {
+				_, err = w.Write([]byte(content))
+			}
+		}
+
+		base := filepath.Base(filename)
+		part("advisory", base, "application/json", doc)
+		part("validation_status", "", "text/plain", string(status))
+		if sha256 != "" {
+			part("hash-256", replaceExt(base, ".sha256"), "text/plain", sha256)
+		}
+		if sha512 != "" {
+			part("hash-512", replaceExt(base, ".sha512"), "text/plain", sha512)
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		if err := writer.Close(); err != nil {
+			return nil, err
+		}
+
+		req, err := http.NewRequest(http.MethodPost, f.cfg.ForwardURL, body)
+		if err != nil {
+			return nil, err
+		}
+		contentType := writer.FormDataContentType()
+		req.Header.Set("Content-Type", contentType)
+		return req, nil
+	}
+
+	// Run this in the main loop of the forwarder.
+	f.cmds <- func(f *forwarder) {
+		req, err := buildRequest()
+		if err != nil {
+			// TODO: improve logging
+			log.Printf("error: %v\n", err)
+			return
+		}
+		res, err := f.httpClient().Do(req)
+		if err != nil {
+			// TODO: improve logging
+			log.Printf("error: %v\n", err)
+			return
+		}
+		if res.StatusCode != http.StatusCreated {
+			// TODO: improve logging
+			defer res.Body.Close()
+			var msg strings.Builder
+			io.Copy(&msg, io.LimitReader(res.Body, 512))
+			var dots string
+			if msg.Len() >= 512 {
+				dots = "..."
+			}
+			log.Printf("error: %s: %q (%d)\n",
+				filename, msg.String()+dots, res.StatusCode)
+		} else {
+			log.Printf("info: forwarding %q succeeded\n", filename)
+		}
+	}
+}

cmd/csaf_downloader/main.go

@@ -30,6 +30,13 @@ func run(cfg *config, domains []string) error {
 	ctx, stop := signal.NotifyContext(ctx, os.Interrupt)
 	defer stop()
 
+	if cfg.ForwardURL != "" {
+		f := newForwarder(cfg)
+		go f.run()
+		defer f.close()
+		d.forwarder = f
+	}
+
 	return d.run(ctx, domains)
 }
 

cmd/csaf_uploader/processor.go

@@ -18,7 +18,6 @@ import (
 	"log"
 	"mime/multipart"
 	"net/http"
-	"net/textproto"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,6 +27,7 @@ import (
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 
 	"github.com/csaf-poc/csaf_distribution/v2/csaf"
+	"github.com/csaf-poc/csaf_distribution/v2/internal/misc"
 	"github.com/csaf-poc/csaf_distribution/v2/util"
 )
 
@@ -103,20 +103,6 @@ func (p *processor) create() error {
 	return nil
 }
 
-var escapeQuotes = strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace
-
-// createFormFile creates an [io.Writer] like [mime/multipart.Writer.CreateFromFile].
-// This version allows to set the mime type, too.
-func createFormFile(w *multipart.Writer, fieldname, filename, mimeType string) (io.Writer, error) {
-	// Source: https://cs.opensource.google/go/go/+/refs/tags/go1.20:src/mime/multipart/writer.go;l=140
-	h := make(textproto.MIMEHeader)
-	h.Set("Content-Disposition",
-		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
-			escapeQuotes(fieldname), escapeQuotes(filename)))
-	h.Set("Content-Type", mimeType)
-	return w.CreatePart(h)
-}
-
 // uploadRequest creates the request for uploading a csaf document by passing the filename.
 // According to the flags values the multipart sections of the request are established.
 // It returns the created http request.
@@ -151,7 +137,7 @@ func (p *processor) uploadRequest(filename string) (*http.Request, error) {
 
 	// As the csaf_provider only accepts uploads with mime type
 	// "application/json" we have to set this.
-	part, err := createFormFile(
+	part, err := misc.CreateFormFile(
 		writer, "csaf", filepath.Base(filename), "application/json")
 	if err != nil {
 		return nil, err

docs/csaf_downloader.md

@@ -7,28 +7,34 @@ A tool to download CSAF documents from CSAF providers.
 csaf_downloader [OPTIONS] domain...
 
 Application Options:
-  -d, --directory=DIR                   DIRectory to store the downloaded files in
-      --insecure                        Do not check TLS certificates from provider
-      --ignoresigcheck                  Ignore signature check results, just warn on mismatch
-      --client-cert=CERT-FILE           TLS client certificate file (PEM encoded data)
-      --client-key=KEY-FILE             TLS client private key file (PEM encoded data)
-      --client-passphrase=PASSPHRASE    Optional passphrase for the client cert (limited, experimental, see doc)
-      --version                         Display version of the binary
-  -v, --verbose                         Verbose output
-  -r, --rate=                           The average upper limit of https operations per second (defaults to
-                                        unlimited)
-  -w, --worker=NUM                      NUMber of concurrent downloads (default: 2)
-  -t, --timerange=RANGE                 RANGE of time from which advisories to download
-  -f, --folder=FOLDER                   Download into a given subFOLDER
-  -i, --ignorepattern=PATTERN           Do not download files if their URLs match any of the given PATTERNs
-  -H, --header=                         One or more extra HTTP header fields
-      --validator=URL                   URL to validate documents remotely
-      --validatorcache=FILE             FILE to cache remote validations
-      --validatorpreset=PRESETS         One or more PRESETS to validate remotely (default: [mandatory])
-  -c, --config=TOML-FILE                Path to config TOML file
+  -d, --directory=DIR                         DIRectory to store the downloaded files in
+      --insecure                              Do not check TLS certificates from provider
+      --ignoresigcheck                        Ignore signature check results, just warn on mismatch
+      --client-cert=CERT-FILE                 TLS client certificate file (PEM encoded data)
+      --client-key=KEY-FILE                   TLS client private key file (PEM encoded data)
+      --client-passphrase=PASSPHRASE          Optional passphrase for the client cert (limited, experimental, see doc)
+      --version                               Display version of the binary
+  -v, --verbose                               Verbose output
+  -n, --nostore                               Do not store files
+  -r, --rate=                                 The average upper limit of https operations per second (defaults to
+                                              unlimited)
+  -w, --worker=NUM                            NUMber of concurrent downloads (default: 2)
+  -t, --timerange=RANGE                       RANGE of time from which advisories to download
+  -f, --folder=FOLDER                         Download into a given subFOLDER
+  -i, --ignorepattern=PATTERN                 Do not download files if their URLs match any of the given PATTERNs
+  -H, --header=                               One or more extra HTTP header fields
+      --validator=URL                         URL to validate documents remotely
+      --validatorcache=FILE                   FILE to cache remote validations
+      --validatorpreset=PRESETS               One or more PRESETS to validate remotely (default: [mandatory])
+  -m, --validationmode=MODE[strict|unsafe]    MODE how strict the validation is (default: strict)
+      --forwardurl=URL                        URL of HTTP endpoint to forward downloads to
+      --forwardheader=                        One or more extra HTTP header fields used by forwarding
+      --forwardqueue=LENGTH                   Maximal queue LENGTH before forwarder
+      --forwardinsecure                       Do not check TLS certificates from forward endpoint
+  -c, --config=TOML-FILE                      Path to config TOML file
 
 Help Options:
-  -h, --help                            Show this help message
+  -h, --help                                  Show this help message
 ```
 
 Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider.
@@ -67,6 +73,11 @@ worker              = 2
 # validator         # not set by default
 # validatorcache    # not set by default
 validatorpreset     = ["mandatory"]
+validation_mode     = "strict"
+# forward_url       # not set by default
+# forward_header    # not set by default
+forward_queue       = 5
+forward_insecure    = false
 ```
 
 The `timerange` parameter enables downloading advisories which last changes falls

go.mod

@@ -1,6 +1,6 @@
 module github.com/csaf-poc/csaf_distribution/v2
 
-go 1.20
+go 1.21
 
 require (
 	github.com/BurntSushi/toml v1.3.2

internal/misc/doc.go

@@ -0,0 +1,10 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
+
+// Package misc implements miscellaneous helper functions.
+package misc

internal/misc/mime.go

@@ -0,0 +1,31 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
+
+package misc
+
+import (
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/textproto"
+	"strings"
+)
+
+var escapeQuotes = strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace
+
+// CreateFormFile creates an [io.Writer] like [mime/multipart.Writer.CreateFromFile].
+// This version allows to set the mime type, too.
+func CreateFormFile(w *multipart.Writer, fieldname, filename, mimeType string) (io.Writer, error) {
+	// Source: https://cs.opensource.google/go/go/+/refs/tags/go1.20:src/mime/multipart/writer.go;l=140
+	h := make(textproto.MIMEHeader)
+	h.Set("Content-Disposition",
+		fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
+			escapeQuotes(fieldname), escapeQuotes(filename)))
+	h.Set("Content-Type", mimeType)
+	return w.CreatePart(h)
+}