From 8c8ccf6a2e7dbd746b73c0e564eb3fafc432e58c Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Sun, 22 Jan 2023 20:55:26 +0100
Subject: [PATCH 01/11] Extract more than one string from expr: category
 fields.

---
 cmd/csaf_aggregator/mirror.go |  9 ++++---
 cmd/csaf_provider/actions.go  |  5 ++--
 docs/csaf_provider.md         |  6 +++++
 util/json.go                  | 50 +++++++++++++++++++++++++++++++++++
 4 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go
index 7fe71d7..3ea4837 100644
--- a/cmd/csaf_aggregator/mirror.go
+++ b/cmd/csaf_aggregator/mirror.go
@@ -451,18 +451,19 @@ func (w *worker) extractCategories(label string, advisory any) error {
 		w.categories[label] = cats
 	}
 
-	var result string
-	matcher := util.StringMatcher(&result)
-
 	const exprPrefix = "expr:"
 
 	for _, cat := range categories {
 		if strings.HasPrefix(cat, exprPrefix) {
 			expr := cat[len(exprPrefix):]
+			var results []string
+			matcher := util.StringTreeMatcher(&results)
 			if err := w.expr.Extract(expr, matcher, true, advisory); err != nil {
 				return err
 			}
-			cats[result] = true
+			for _, result := range results {
+				cats[result] = true
+			}
 		} else { // Normal
 			cats[cat] = true
 		}
diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go
index ea099e3..b1ad0a9 100644
--- a/cmd/csaf_provider/actions.go
+++ b/cmd/csaf_provider/actions.go
@@ -195,9 +195,10 @@ func (c *controller) upload(r *http.Request) (any, error) {
 	var dynamicCategories []string
 	if catExprs := c.cfg.DynamicCategories(); len(catExprs) > 0 {
 		var err error
-		if dynamicCategories, err = pe.Strings(catExprs, true, content); err != nil {
+		if dynamicCategories, err = pe.StringsFromTree(
+			catExprs, true, content); err != nil {
 			// XXX: Should we die here?
-			log.Printf("eval of dynamic catecory expressions failed: %v\n", err)
+			log.Printf("eval of dynamic catergory expressions failed: %v\n", err)
 		}
 	}
 
diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md
index 81a45fa..464f0db 100644
--- a/docs/csaf_provider.md
+++ b/docs/csaf_provider.md
@@ -106,6 +106,12 @@ The following example file documents all available configuration options:
 # If a list item starts with `expr:`
 #   the rest of the string is used as a JsonPath expression
 #   to extract a string from the incoming advisories.
+#   If the result of the expression is a string this string
+#   is used. If the result is an array each element of
+#   this array is tested if it is a string or an array.
+#   If this test fails the expression fails. If the
+#   test succeeds the rules are applied recursively to
+#   collect all strings in the result.
 # Strings not starting with `expr:` are taken verbatim.
 # By default no category documents are created.
 # This example provides an overview over the syntax,
diff --git a/util/json.go b/util/json.go
index ed0fd70..48d0e09 100644
--- a/util/json.go
+++ b/util/json.go
@@ -101,6 +101,37 @@ func StringMatcher(dst *string) func(any) error {
 	}
 }
 
+// StringTreeMatcher returns a matcher which add strings from
+// stringss and recursively from arrays from strings.
+func StringTreeMatcher(strings *[]string) func(any) error {
+	// Only add unique strings.
+	unique := func(s string) {
+		for _, t := range *strings {
+			if s == t {
+				return
+			}
+		}
+		*strings = append(*strings, s)
+	}
+	var recurse func(any) error
+	recurse = func(x any) error {
+		switch y := x.(type) {
+		case string:
+			unique(y)
+		case []any:
+			for _, z := range y {
+				if err := recurse(z); err != nil {
+					return err
+				}
+			}
+		default:
+			return fmt.Errorf("unsupported type: %T", x)
+		}
+		return nil
+	}
+	return recurse
+}
+
 // TimeMatcher stores a time with a given format.
 func TimeMatcher(dst *time.Time, format string) func(any) error {
 	return func(x any) error {
@@ -147,6 +178,25 @@ func (pe *PathEval) Match(matcher []PathEvalMatcher, doc any) error {
 	return nil
 }
 
+// StringsFromTree returns strings from the given exprs.
+//  1. If a expression results a string this string is used.
+//  2. if a expression results in an array the elements
+//     of this array are recursively treated with 1. and 2.
+func (pe *PathEval) StringsFromTree(
+	exprs []string,
+	optional bool,
+	doc any,
+) ([]string, error) {
+	results := make([]string, 0, len(exprs))
+	matcher := StringTreeMatcher(&results)
+	for _, expr := range exprs {
+		if err := pe.Extract(expr, matcher, optional, doc); err != nil {
+			return nil, err
+		}
+	}
+	return results, nil
+}
+
 // Strings searches the given document for the given set of expressions
 // and returns the corresponding strings. The optional flag indicates
 // if the expression evaluation have to succseed or not.

From 6bf8c530c6f8f2b02c5b58121c262a4fdb5c6498 Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Mon, 23 Jan 2023 14:30:47 +0100
Subject: [PATCH 02/11] Fix comment typo

---
 util/json.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/util/json.go b/util/json.go
index 48d0e09..d21b4cf 100644
--- a/util/json.go
+++ b/util/json.go
@@ -101,8 +101,8 @@ func StringMatcher(dst *string) func(any) error {
 	}
 }
 
-// StringTreeMatcher returns a matcher which add strings from
-// stringss and recursively from arrays from strings.
+// StringTreeMatcher returns a matcher which addis strings from
+// strings and recursively from arrays of strings.
 func StringTreeMatcher(strings *[]string) func(any) error {
 	// Only add unique strings.
 	unique := func(s string) {

From bf6dfafffdf79f2697e7a00912fe09e337b8b3ee Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Tue, 24 Jan 2023 01:34:23 +0100
Subject: [PATCH 03/11] Address remarks from review.

---
 docs/csaf_provider.md | 10 ++++++++++
 util/json.go          |  8 ++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md
index 464f0db..4db4d31 100644
--- a/docs/csaf_provider.md
+++ b/docs/csaf_provider.md
@@ -112,6 +112,16 @@ The following example file documents all available configuration options:
 #   If this test fails the expression fails. If the
 #   test succeeds the rules are applied recursively to
 #   collect all strings in the result.
+#   If the result of the expression is a string this string
+#   is used. If the result is an array each element of
+#   this array is tested if it is a string or an array.
+#   If this test fails the expression fails. If the
+#   test succeeds the rules are applied recursively to
+#   collect all strings in the result.
+#   Suggested expressions are:
+#   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category=='vendor' || @.category=='product_family' || @.category=='product_name')].name"
+#   - CVEs: "expr:$.vulnerabilities[*].cve"
+#   - CWEs: "expr:$.vulnerabilities[*].cwe.id"
 # Strings not starting with `expr:` are taken verbatim.
 # By default no category documents are created.
 # This example provides an overview over the syntax,
diff --git a/util/json.go b/util/json.go
index d21b4cf..985c7e1 100644
--- a/util/json.go
+++ b/util/json.go
@@ -101,8 +101,8 @@ func StringMatcher(dst *string) func(any) error {
 	}
 }
 
-// StringTreeMatcher returns a matcher which addis strings from
-// strings and recursively from arrays of strings.
+// StringTreeMatcher returns a matcher which adds strings
+// to a slice and recursively strings from arrays of strings.
 func StringTreeMatcher(strings *[]string) func(any) error {
 	// Only add unique strings.
 	unique := func(s string) {
@@ -179,8 +179,8 @@ func (pe *PathEval) Match(matcher []PathEvalMatcher, doc any) error {
 }
 
 // StringsFromTree returns strings from the given exprs.
-//  1. If a expression results a string this string is used.
-//  2. if a expression results in an array the elements
+//  1. If an expression results in a string this string is used.
+//  2. if an expression results in an array the elements
 //     of this array are recursively treated with 1. and 2.
 func (pe *PathEval) StringsFromTree(
 	exprs []string,

From 6dedeff7fc0b8635f95f6440ab7e0b19d53c128f Mon Sep 17 00:00:00 2001
From: JanHoefelmeyer <107021473+JanHoefelmeyer@users.noreply.github.com>
Date: Wed, 25 Jan 2023 11:40:25 +0100
Subject: [PATCH 04/11] Update actions.go

Fixes typo: catergory -> category
---
 cmd/csaf_provider/actions.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go
index b1ad0a9..67b36a5 100644
--- a/cmd/csaf_provider/actions.go
+++ b/cmd/csaf_provider/actions.go
@@ -198,7 +198,7 @@ func (c *controller) upload(r *http.Request) (any, error) {
 		if dynamicCategories, err = pe.StringsFromTree(
 			catExprs, true, content); err != nil {
 			// XXX: Should we die here?
-			log.Printf("eval of dynamic catergory expressions failed: %v\n", err)
+			log.Printf("eval of dynamic category expressions failed: %v\n", err)
 		}
 	}
 

From 69dda45bac1ae7ae4733a3b4c0e9caa8d160cb51 Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Wed, 25 Jan 2023 13:42:26 +0100
Subject: [PATCH 05/11] Dedup doc

---
 docs/csaf_provider.md | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md
index 4db4d31..177e71d 100644
--- a/docs/csaf_provider.md
+++ b/docs/csaf_provider.md
@@ -112,12 +112,6 @@ The following example file documents all available configuration options:
 #   If this test fails the expression fails. If the
 #   test succeeds the rules are applied recursively to
 #   collect all strings in the result.
-#   If the result of the expression is a string this string
-#   is used. If the result is an array each element of
-#   this array is tested if it is a string or an array.
-#   If this test fails the expression fails. If the
-#   test succeeds the rules are applied recursively to
-#   collect all strings in the result.
 #   Suggested expressions are:
 #   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category=='vendor' || @.category=='product_family' || @.category=='product_name')].name"
 #   - CVEs: "expr:$.vulnerabilities[*].cve"

From 7e9ea2c9acfb285260342c857de3cb00b2d07a54 Mon Sep 17 00:00:00 2001
From: Bernhard Reiter <bernhard@intevation.de>
Date: Thu, 26 Jan 2023 11:56:51 +0100
Subject: [PATCH 06/11] =?UTF-8?q?Improve=20setup=20script=20f=C3=BCr=20Ubu?=
 =?UTF-8?q?ntu=20systems?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

 * Make sure that we do not have an old go installation around,
   because otherwise it maybe broken by untarring the newer one.
   As documented on https://go.dev/doc/install .
---
 docs/scripts/prepareUbuntuInstanceForITests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/scripts/prepareUbuntuInstanceForITests.sh b/docs/scripts/prepareUbuntuInstanceForITests.sh
index a062bbe..8811c53 100755
--- a/docs/scripts/prepareUbuntuInstanceForITests.sh
+++ b/docs/scripts/prepareUbuntuInstanceForITests.sh
@@ -11,4 +11,5 @@ apt install -y make bash sed tar git nginx fcgiwrap gnutls-bin
 # Install Go from binary distribution
 latest_go="$(curl https://go.dev/VERSION\?m=text).linux-amd64.tar.gz"
 curl -O https://dl.google.com/go/$latest_go
+rm -rf /usr/local/go # be sure that we do not have an old installation
 tar -C /usr/local -xzf $latest_go

From ad5c678abf7d4faa164b827fffcdfdb79bea52ae Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Thu, 26 Jan 2023 18:55:21 +0100
Subject: [PATCH 07/11] Mention the single quote limitation of the jsonpath
 implementation

---
 docs/csaf_provider.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md
index 177e71d..c0b8f3d 100644
--- a/docs/csaf_provider.md
+++ b/docs/csaf_provider.md
@@ -113,9 +113,10 @@ The following example file documents all available configuration options:
 #   test succeeds the rules are applied recursively to
 #   collect all strings in the result.
 #   Suggested expressions are:
-#   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category=='vendor' || @.category=='product_family' || @.category=='product_name')].name"
+#   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name"
 #   - CVEs: "expr:$.vulnerabilities[*].cve"
 #   - CWEs: "expr:$.vulnerabilities[*].cwe.id"
+#  s implementation of JsonPath expressions does not support the use of single-quotes. Double quotes have to be quoted. 
 # Strings not starting with `expr:` are taken verbatim.
 # By default no category documents are created.
 # This example provides an overview over the syntax,

From ff31ebfa0fac198305788b90e77822281258695c Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Thu, 26 Jan 2023 20:02:05 +0100
Subject: [PATCH 08/11] Fixed typos

---
 docs/csaf_provider.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md
index c0b8f3d..d1e6b22 100644
--- a/docs/csaf_provider.md
+++ b/docs/csaf_provider.md
@@ -116,7 +116,8 @@ The following example file documents all available configuration options:
 #   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name"
 #   - CVEs: "expr:$.vulnerabilities[*].cve"
 #   - CWEs: "expr:$.vulnerabilities[*].cwe.id"
-#  s implementation of JsonPath expressions does not support the use of single-quotes. Double quotes have to be quoted. 
+#   The used implementation to evaluate JSONPath expressions does
+#   not support the use of single-quotes. Double quotes have to be quoted. 
 # Strings not starting with `expr:` are taken verbatim.
 # By default no category documents are created.
 # This example provides an overview over the syntax,

From 0745a0943d4050e864f00f9fc83a075d32d51d60 Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Thu, 26 Jan 2023 21:54:46 +0100
Subject: [PATCH 09/11] Separate compiling and evaluation of dynamic
 categories.

---
 cmd/csaf_aggregator/mirror.go | 22 +++++++++++++++-------
 cmd/csaf_provider/actions.go  | 16 +++++++++++-----
 util/json.go                  | 33 ++++++++++++++-------------------
 3 files changed, 40 insertions(+), 31 deletions(-)

diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go
index 3ea4837..fd37b3e 100644
--- a/cmd/csaf_aggregator/mirror.go
+++ b/cmd/csaf_aggregator/mirror.go
@@ -453,22 +453,30 @@ func (w *worker) extractCategories(label string, advisory any) error {
 
 	const exprPrefix = "expr:"
 
+	var dynamic []string
+	matcher := util.StringTreeMatcher(&dynamic)
+
 	for _, cat := range categories {
 		if strings.HasPrefix(cat, exprPrefix) {
 			expr := cat[len(exprPrefix):]
-			var results []string
-			matcher := util.StringTreeMatcher(&results)
-			if err := w.expr.Extract(expr, matcher, true, advisory); err != nil {
-				return err
-			}
-			for _, result := range results {
-				cats[result] = true
+			// Compile first to check that the expression is okay.
+			if _, err := w.expr.Compile(expr); err != nil {
+				fmt.Printf("Compiling category expression %q failed: %v\n",
+					expr, err)
+				continue
 			}
+			// Ignore errors here as they result from not matching.
+			w.expr.Extract(expr, matcher, true, advisory)
 		} else { // Normal
 			cats[cat] = true
 		}
 	}
 
+	// Add dynamic categories.
+	for _, cat := range dynamic {
+		cats[cat] = true
+	}
+
 	return nil
 }
 
diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go
index 67b36a5..0fa4e5b 100644
--- a/cmd/csaf_provider/actions.go
+++ b/cmd/csaf_provider/actions.go
@@ -194,11 +194,17 @@ func (c *controller) upload(r *http.Request) (any, error) {
 	// Check if we have to search for dynamic categories.
 	var dynamicCategories []string
 	if catExprs := c.cfg.DynamicCategories(); len(catExprs) > 0 {
-		var err error
-		if dynamicCategories, err = pe.StringsFromTree(
-			catExprs, true, content); err != nil {
-			// XXX: Should we die here?
-			log.Printf("eval of dynamic category expressions failed: %v\n", err)
+		matcher := util.StringTreeMatcher(&dynamicCategories)
+
+		for _, expr := range catExprs {
+			// Compile first to check that the expression is okay.
+			if _, err := pe.Compile(expr); err != nil {
+				log.Printf("Compiling category expression %q failed: %v\n",
+					expr, err)
+				continue
+			}
+			// Ignore errors here as they result from not matching.
+			pe.Extract(expr, matcher, true, content)
 		}
 	}
 
diff --git a/util/json.go b/util/json.go
index 985c7e1..be9f330 100644
--- a/util/json.go
+++ b/util/json.go
@@ -42,6 +42,20 @@ func NewPathEval() *PathEval {
 	}
 }
 
+// Compile compiles an expression and stores it in the
+// internal cache on success.
+func (pe *PathEval) Compile(expr string) (gval.Evaluable, error) {
+	if eval := pe.exprs[expr]; eval != nil {
+		return eval, nil
+	}
+	eval, err := pe.builder.NewEvaluable(expr)
+	if err != nil {
+		return nil, err
+	}
+	pe.exprs[expr] = eval
+	return eval, nil
+}
+
 // Eval evalutes expression expr on document doc.
 // Returns the result of the expression.
 func (pe *PathEval) Eval(expr string, doc any) (any, error) {
@@ -178,25 +192,6 @@ func (pe *PathEval) Match(matcher []PathEvalMatcher, doc any) error {
 	return nil
 }
 
-// StringsFromTree returns strings from the given exprs.
-//  1. If an expression results in a string this string is used.
-//  2. if an expression results in an array the elements
-//     of this array are recursively treated with 1. and 2.
-func (pe *PathEval) StringsFromTree(
-	exprs []string,
-	optional bool,
-	doc any,
-) ([]string, error) {
-	results := make([]string, 0, len(exprs))
-	matcher := StringTreeMatcher(&results)
-	for _, expr := range exprs {
-		if err := pe.Extract(expr, matcher, optional, doc); err != nil {
-			return nil, err
-		}
-	}
-	return results, nil
-}
-
 // Strings searches the given document for the given set of expressions
 // and returns the corresponding strings. The optional flag indicates
 // if the expression evaluation have to succseed or not.

From 51fba46893f04646474e7d63309e6696f589afcc Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Wed, 25 Jan 2023 16:45:05 +0100
Subject: [PATCH 10/11] Add extra http header support to downloader and
 checker.

---
 cmd/csaf_checker/main.go          | 20 ++++++-----
 cmd/csaf_checker/processor.go     | 30 +++++++++-------
 cmd/csaf_downloader/downloader.go | 30 +++++++++-------
 cmd/csaf_downloader/main.go       | 12 ++++---
 docs/csaf_checker.md              |  3 +-
 docs/csaf_downloader.md           |  4 +--
 util/client.go                    | 59 +++++++++++++++++++++++++++++++
 7 files changed, 117 insertions(+), 41 deletions(-)

diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go
index 78f58b6..476851a 100644
--- a/cmd/csaf_checker/main.go
+++ b/cmd/csaf_checker/main.go
@@ -19,6 +19,7 @@ import (
 	"html/template"
 	"io"
 	"log"
+	"net/http"
 	"os"
 
 	"github.com/csaf-poc/csaf_distribution/util"
@@ -29,15 +30,16 @@ import (
 var reportHTML string
 
 type options struct {
-	Output     string   `short:"o" long:"output" description:"File name of the generated report" value-name:"REPORT-FILE"`
-	Format     string   `short:"f" long:"format" choice:"json" choice:"html" description:"Format of report" default:"json"`
-	Insecure   bool     `long:"insecure" description:"Do not check TLS certificates from provider"`
-	ClientCert *string  `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"`
-	ClientKey  *string  `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"`
-	Version    bool     `long:"version" description:"Display version of the binary"`
-	Verbose    bool     `long:"verbose" short:"v" description:"Verbose output"`
-	Rate       *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
-	Years      *uint    `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"`
+	Output      string      `short:"o" long:"output" description:"File name of the generated report" value-name:"REPORT-FILE"`
+	Format      string      `short:"f" long:"format" choice:"json" choice:"html" description:"Format of report" default:"json"`
+	Insecure    bool        `long:"insecure" description:"Do not check TLS certificates from provider"`
+	ClientCert  *string     `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"`
+	ClientKey   *string     `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"`
+	Version     bool        `long:"version" description:"Display version of the binary"`
+	Verbose     bool        `long:"verbose" short:"v" description:"Verbose output"`
+	Rate        *float64    `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
+	Years       *uint       `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"`
+	ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
 
 	clientCerts []tls.Certificate
 }
diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go
index 32eb991..46591ad 100644
--- a/cmd/csaf_checker/processor.go
+++ b/cmd/csaf_checker/processor.go
@@ -351,24 +351,30 @@ func (p *processor) httpClient() util.Client {
 		TLSClientConfig: &tlsConfig,
 	}
 
-	var client util.Client
+	client := util.Client(&hClient)
 
+	// Add extra headers.
+	if len(p.opts.ExtraHeader) > 0 {
+		client = &util.HeaderClient{
+			Client: client,
+			Header: p.opts.ExtraHeader,
+		}
+	}
+
+	// Add optional URL logging.
 	if p.opts.Verbose {
-		client = &util.LoggingClient{Client: &hClient}
-	} else {
-		client = &hClient
+		client = &util.LoggingClient{Client: client}
 	}
 
-	if p.opts.Rate == nil {
-		p.client = client
-		return client
-	}
-
-	p.client = &util.LimitingClient{
-		Client:  client,
-		Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1),
+	// Add optional rate limiting.
+	if p.opts.Rate != nil {
+		client = &util.LimitingClient{
+			Client:  client,
+			Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1),
+		}
 	}
 
+	p.client = client
 	return p.client
 }
 
diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go
index 6674ef7..3bb2f93 100644
--- a/cmd/csaf_downloader/downloader.go
+++ b/cmd/csaf_downloader/downloader.go
@@ -64,24 +64,30 @@ func (d *downloader) httpClient() util.Client {
 		}
 	}
 
-	var client util.Client
+	client := util.Client(&hClient)
 
+	// Add extra headers.
+	if len(d.opts.ExtraHeader) > 0 {
+		client = &util.HeaderClient{
+			Client: client,
+			Header: d.opts.ExtraHeader,
+		}
+	}
+
+	// Add optional URL logging.
 	if d.opts.Verbose {
-		client = &util.LoggingClient{Client: &hClient}
-	} else {
-		client = &hClient
+		client = &util.LoggingClient{Client: client}
 	}
 
-	if d.opts.Rate == nil {
-		d.client = client
-		return client
-	}
-
-	d.client = &util.LimitingClient{
-		Client:  client,
-		Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1),
+	// Add optional rate limiting.
+	if d.opts.Rate != nil {
+		client = &util.LimitingClient{
+			Client:  client,
+			Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1),
+		}
 	}
 
+	d.client = client
 	return d.client
 }
 
diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go
index 58ebe5e..38afe7d 100644
--- a/cmd/csaf_downloader/main.go
+++ b/cmd/csaf_downloader/main.go
@@ -12,6 +12,7 @@ package main
 import (
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 
 	"github.com/csaf-poc/csaf_distribution/util"
@@ -19,11 +20,12 @@ import (
 )
 
 type options struct {
-	Directory *string  `short:"d" long:"directory" description:"Directory to store the downloaded files in"`
-	Insecure  bool     `long:"insecure" description:"Do not check TLS certificates from provider"`
-	Version   bool     `long:"version" description:"Display version of the binary"`
-	Verbose   bool     `long:"verbose" short:"v" description:"Verbose output"`
-	Rate      *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
+	Directory   *string     `short:"d" long:"directory" description:"Directory to store the downloaded files in"`
+	Insecure    bool        `long:"insecure" description:"Do not check TLS certificates from provider"`
+	Version     bool        `long:"version" description:"Display version of the binary"`
+	Verbose     bool        `long:"verbose" short:"v" description:"Verbose output"`
+	Rate        *float64    `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
+	ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
 }
 
 func errCheck(err error) {
diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md
index 39c265d..5e134c0 100644
--- a/docs/csaf_checker.md
+++ b/docs/csaf_checker.md
@@ -15,13 +15,14 @@ Application Options:
   -v, --verbose                  Verbose output
   -r, --rate=                    The average upper limit of https operations per second
   -y, --years=YEARS              Number of years to look back from now
+  -H, --header=                  One or more extra HTTP header fields
 
 Help Options:
   -h, --help                     Show this help message
 ```
 
 Usage example:
-` ./csaf_checker example.com -f html --rate=5.3 -o check-results.html`
+` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
 
 Each performed check has a return type of either 0,1 or 2:
 ```
diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md
index c88a35b..56d54ce 100644
--- a/docs/csaf_downloader.md
+++ b/docs/csaf_downloader.md
@@ -4,8 +4,7 @@ A tool to download CSAF content from a specific domain/provider.
 ### Usage
 
 ```
-Usage:
-  csaf_downloader [OPTIONS] domain...
+csaf_downloader [OPTIONS] domain...
 
 Application Options:
   -d, --directory= Directory to store the downloaded files in
@@ -13,6 +12,7 @@ Application Options:
       --version    Display version of the binary
   -v, --verbose    Verbose output
   -r, --rate=      The average upper limit of https operations per second
+  -H, --header=    One or more extra HTTP header fields
 
 Help Options:
   -h, --help       Show this help message
diff --git a/util/client.go b/util/client.go
index d6cd150..2a2b32e 100644
--- a/util/client.go
+++ b/util/client.go
@@ -14,6 +14,7 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"strings"
 
 	"golang.org/x/time/rate"
 )
@@ -38,6 +39,64 @@ type LimitingClient struct {
 	Limiter *rate.Limiter
 }
 
+// HeaderClient adds extra HTTP header fields to the ou going requests.
+type HeaderClient struct {
+	Client
+	Header http.Header
+}
+
+// Do implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) {
+	// Maybe this overly careful but this minimizes
+	// potential side effects in the caller.
+	orig := req.Header
+	defer func() { req.Header = orig }()
+
+	// Work on a copy.
+	req.Header = req.Header.Clone()
+
+	for key, values := range hc.Header {
+		for _, v := range values {
+			req.Header.Add(key, v)
+		}
+	}
+	return hc.Client.Do(req)
+}
+
+// Get implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Get(url string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return hc.Do(req)
+}
+
+// Head implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Head(url string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return hc.Do(req)
+}
+
+// Post implements the respective method of the [Client] interface.
+func (hc *HeaderClient) Post(url, contentType string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", contentType)
+	return hc.Do(req)
+}
+
+// PostForm implements the respective method of the [Client] interface.
+func (hc *HeaderClient) PostForm(url string, data url.Values) (*http.Response, error) {
+	return hc.Post(
+		url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
+}
+
 // Do implements the respective method of the Client interface.
 func (lc *LoggingClient) Do(req *http.Request) (*http.Response, error) {
 	log.Printf("[DO]: %s\n", req.URL.String())

From 18e6fee2e75d6498b1019be83a12f79e2b643c5e Mon Sep 17 00:00:00 2001
From: "Sascha L. Teichmann" <sascha.teichmann@intevation.de>
Date: Wed, 25 Jan 2023 17:04:00 +0100
Subject: [PATCH 11/11] Fix typo in comment

---
 util/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/util/client.go b/util/client.go
index 2a2b32e..60a5bd0 100644
--- a/util/client.go
+++ b/util/client.go
@@ -39,7 +39,7 @@ type LimitingClient struct {
 	Limiter *rate.Limiter
 }
 
-// HeaderClient adds extra HTTP header fields to the ou going requests.
+// HeaderClient adds extra HTTP header fields to requests.
 type HeaderClient struct {
 	Client
 	Header http.Header