Merged main into branch

cmd/csaf_aggregator/mirror.go

@@ -451,23 +451,32 @@ func (w *worker) extractCategories(label string, advisory any) error {
 		w.categories[label] = cats
 	}
 
-	var result string
-	matcher := util.StringMatcher(&result)
-
 	const exprPrefix = "expr:"
 
+	var dynamic []string
+	matcher := util.StringTreeMatcher(&dynamic)
+
 	for _, cat := range categories {
 		if strings.HasPrefix(cat, exprPrefix) {
 			expr := cat[len(exprPrefix):]
-			if err := w.expr.Extract(expr, matcher, true, advisory); err != nil {
+			// Compile first to check that the expression is okay.
-				return err
+			if _, err := w.expr.Compile(expr); err != nil {
+				fmt.Printf("Compiling category expression %q failed: %v\n",
+					expr, err)
+				continue
 			}
-			cats[result] = true
+			// Ignore errors here as they result from not matching.
+			w.expr.Extract(expr, matcher, true, advisory)
 		} else { // Normal
 			cats[cat] = true
 		}
 	}
 
+	// Add dynamic categories.
+	for _, cat := range dynamic {
+		cats[cat] = true
+	}
+
 	return nil
 }
 

cmd/csaf_checker/main.go

@@ -19,6 +19,7 @@ import (
 	"html/template"
 	"io"
 	"log"
+	"net/http"
 	"os"
 
 	"github.com/csaf-poc/csaf_distribution/util"
@@ -29,15 +30,16 @@ import (
 var reportHTML string
 
 type options struct {
-	Output     string   `short:"o" long:"output" description:"File name of the generated report" value-name:"REPORT-FILE"`
+	Output      string      `short:"o" long:"output" description:"File name of the generated report" value-name:"REPORT-FILE"`
-	Format     string   `short:"f" long:"format" choice:"json" choice:"html" description:"Format of report" default:"json"`
+	Format      string      `short:"f" long:"format" choice:"json" choice:"html" description:"Format of report" default:"json"`
-	Insecure   bool     `long:"insecure" description:"Do not check TLS certificates from provider"`
+	Insecure    bool        `long:"insecure" description:"Do not check TLS certificates from provider"`
-	ClientCert *string  `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"`
+	ClientCert  *string     `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"`
-	ClientKey  *string  `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"`
+	ClientKey   *string     `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"`
-	Version    bool     `long:"version" description:"Display version of the binary"`
+	Version     bool        `long:"version" description:"Display version of the binary"`
-	Verbose    bool     `long:"verbose" short:"v" description:"Verbose output"`
+	Verbose     bool        `long:"verbose" short:"v" description:"Verbose output"`
-	Rate       *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
+	Rate        *float64    `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
-	Years      *uint    `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"`
+	Years       *uint       `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"`
+	ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
 
 	clientCerts []tls.Certificate
 }

cmd/csaf_checker/processor.go

@@ -351,24 +351,30 @@ func (p *processor) httpClient() util.Client {
 		TLSClientConfig: &tlsConfig,
 	}
 
-	var client util.Client
+	client := util.Client(&hClient)
 
+	// Add extra headers.
+	if len(p.opts.ExtraHeader) > 0 {
+		client = &util.HeaderClient{
+			Client: client,
+			Header: p.opts.ExtraHeader,
+		}
+	}
+
+	// Add optional URL logging.
 	if p.opts.Verbose {
-		client = &util.LoggingClient{Client: &hClient}
+		client = &util.LoggingClient{Client: client}
-	} else {
-		client = &hClient
 	}
 
-	if p.opts.Rate == nil {
+	// Add optional rate limiting.
-		p.client = client
+	if p.opts.Rate != nil {
-		return client
+		client = &util.LimitingClient{
-	}
+			Client:  client,
-
+			Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1),
-	p.client = &util.LimitingClient{
+		}
-		Client:  client,
-		Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1),
 	}
 
+	p.client = client
 	return p.client
 }
 

cmd/csaf_downloader/downloader.go

@@ -89,24 +89,30 @@ func (d *downloader) httpClient() util.Client {
 		}
 	}
 
-	var client util.Client
+	client := util.Client(&hClient)
 
+	// Add extra headers.
+	if len(d.opts.ExtraHeader) > 0 {
+		client = &util.HeaderClient{
+			Client: client,
+			Header: d.opts.ExtraHeader,
+		}
+	}
+
+	// Add optional URL logging.
 	if d.opts.Verbose {
-		client = &util.LoggingClient{Client: &hClient}
+		client = &util.LoggingClient{Client: client}
-	} else {
-		client = &hClient
 	}
 
-	if d.opts.Rate == nil {
+	// Add optional rate limiting.
-		d.client = client
+	if d.opts.Rate != nil {
-		return client
+		client = &util.LimitingClient{
-	}
+			Client:  client,
-
+			Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1),
-	d.client = &util.LimitingClient{
+		}
-		Client:  client,
-		Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1),
 	}
 
+	d.client = client
 	return d.client
 }
 

cmd/csaf_downloader/main.go

@@ -12,6 +12,7 @@ package main
 import (
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 
 	"github.com/csaf-poc/csaf_distribution/util"
@@ -25,6 +26,8 @@ type options struct {
 	Verbose   bool     `long:"verbose" short:"v" description:"Verbose output"`
 	Rate      *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
 
+	ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
+
 	RemoteValidator        string   `long:"validator" description:"URL to validate documents remotely" value-name:"URL"`
 	RemoteValidatorCache   string   `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"`
 	RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"`

cmd/csaf_provider/actions.go

@@ -194,10 +194,17 @@ func (c *controller) upload(r *http.Request) (any, error) {
 	// Check if we have to search for dynamic categories.
 	var dynamicCategories []string
 	if catExprs := c.cfg.DynamicCategories(); len(catExprs) > 0 {
-		var err error
+		matcher := util.StringTreeMatcher(&dynamicCategories)
-		if dynamicCategories, err = pe.Strings(catExprs, true, content); err != nil {
+
-			// XXX: Should we die here?
+		for _, expr := range catExprs {
-			log.Printf("eval of dynamic catecory expressions failed: %v\n", err)
+			// Compile first to check that the expression is okay.
+			if _, err := pe.Compile(expr); err != nil {
+				log.Printf("Compiling category expression %q failed: %v\n",
+					expr, err)
+				continue
+			}
+			// Ignore errors here as they result from not matching.
+			pe.Extract(expr, matcher, true, content)
 		}
 	}
 

docs/csaf_checker.md

@@ -15,13 +15,14 @@ Application Options:
   -v, --verbose                  Verbose output
   -r, --rate=                    The average upper limit of https operations per second
   -y, --years=YEARS              Number of years to look back from now
+  -H, --header=                  One or more extra HTTP header fields
 
 Help Options:
   -h, --help                     Show this help message
 ```
 
 Usage example:
-` ./csaf_checker example.com -f html --rate=5.3 -o check-results.html`
+` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
 
 Each performed check has a return type of either 0,1 or 2:
 ```

docs/csaf_downloader.md

@@ -4,8 +4,7 @@ A tool to download CSAF content from a specific domain/provider.
 ### Usage
 
 ```
-Usage:
+csaf_downloader [OPTIONS] domain...
-  csaf_downloader [OPTIONS] domain...
 
 Application Options:
   -d, --directory=DIR          DIRectory to store the downloaded files in
@@ -13,6 +12,7 @@ Application Options:
       --version                Display version of the binary
   -v, --verbose                Verbose output
   -r, --rate=                  The average upper limit of https operations per second
+  -H, --header=                One or more extra HTTP header fields
       --validator=URL          URL to validate documents remotely
       --validatorcache=FILE    FILE to cache remote validations
       --validatorpreset=       One or more presets to validate remotely (default: mandatory)

docs/csaf_provider.md

@@ -106,6 +106,18 @@ The following example file documents all available configuration options:
 # If a list item starts with `expr:`
 #   the rest of the string is used as a JsonPath expression
 #   to extract a string from the incoming advisories.
+#   If the result of the expression is a string this string
+#   is used. If the result is an array each element of
+#   this array is tested if it is a string or an array.
+#   If this test fails the expression fails. If the
+#   test succeeds the rules are applied recursively to
+#   collect all strings in the result.
+#   Suggested expressions are:
+#   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name"
+#   - CVEs: "expr:$.vulnerabilities[*].cve"
+#   - CWEs: "expr:$.vulnerabilities[*].cwe.id"
+#   The used implementation to evaluate JSONPath expressions does
+#   not support the use of single-quotes. Double quotes have to be quoted. 
 # Strings not starting with `expr:` are taken verbatim.
 # By default no category documents are created.
 # This example provides an overview over the syntax,

docs/scripts/prepareUbuntuInstanceForITests.sh

@@ -11,4 +11,5 @@ apt install -y make bash sed tar git nginx fcgiwrap gnutls-bin
 # Install Go from binary distribution
 latest_go="$(curl https://go.dev/VERSION\?m=text).linux-amd64.tar.gz"
 curl -O https://dl.google.com/go/$latest_go
+rm -rf /usr/local/go # be sure that we do not have an old installation
 tar -C /usr/local -xzf $latest_go

util/client.go

@@ -14,6 +14,7 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"strings"
 
 	"golang.org/x/time/rate"
 )
@@ -38,6 +39,64 @@ type LimitingClient struct {
 	Limiter *rate.Limiter
 }
 
+// HeaderClient adds extra HTTP header fields to requests.
+type HeaderClient struct {
+	Client
+	Header http.Header
+}
+
+// Do implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) {
+	// Maybe this overly careful but this minimizes
+	// potential side effects in the caller.
+	orig := req.Header
+	defer func() { req.Header = orig }()
+
+	// Work on a copy.
+	req.Header = req.Header.Clone()
+
+	for key, values := range hc.Header {
+		for _, v := range values {
+			req.Header.Add(key, v)
+		}
+	}
+	return hc.Client.Do(req)
+}
+
+// Get implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Get(url string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return hc.Do(req)
+}
+
+// Head implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Head(url string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return hc.Do(req)
+}
+
+// Post implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Post(url, contentType string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", contentType)
+	return hc.Do(req)
+}
+
+// PostForm implements the respective method of the [Client] interface.
+func (hc *HeaderClient) PostForm(url string, data url.Values) (*http.Response, error) {
+	return hc.Post(
+		url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
+}
+
 // Do implements the respective method of the Client interface.
 func (lc *LoggingClient) Do(req *http.Request) (*http.Response, error) {
 	log.Printf("[DO]: %s\n", req.URL.String())

util/json.go

@@ -42,6 +42,20 @@ func NewPathEval() *PathEval {
 	}
 }
 
+// Compile compiles an expression and stores it in the
+// internal cache on success.
+func (pe *PathEval) Compile(expr string) (gval.Evaluable, error) {
+	if eval := pe.exprs[expr]; eval != nil {
+		return eval, nil
+	}
+	eval, err := pe.builder.NewEvaluable(expr)
+	if err != nil {
+		return nil, err
+	}
+	pe.exprs[expr] = eval
+	return eval, nil
+}
+
 // Eval evalutes expression expr on document doc.
 // Returns the result of the expression.
 func (pe *PathEval) Eval(expr string, doc any) (any, error) {
@@ -101,6 +115,37 @@ func StringMatcher(dst *string) func(any) error {
 	}
 }
 
+// StringTreeMatcher returns a matcher which adds strings
+// to a slice and recursively strings from arrays of strings.
+func StringTreeMatcher(strings *[]string) func(any) error {
+	// Only add unique strings.
+	unique := func(s string) {
+		for _, t := range *strings {
+			if s == t {
+				return
+			}
+		}
+		*strings = append(*strings, s)
+	}
+	var recurse func(any) error
+	recurse = func(x any) error {
+		switch y := x.(type) {
+		case string:
+			unique(y)
+		case []any:
+			for _, z := range y {
+				if err := recurse(z); err != nil {
+					return err
+				}
+			}
+		default:
+			return fmt.Errorf("unsupported type: %T", x)
+		}
+		return nil
+	}
+	return recurse
+}
+
 // TimeMatcher stores a time with a given format.
 func TimeMatcher(dst *time.Time, format string) func(any) error {
 	return func(x any) error {