diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go index 7fe71d7..fd37b3e 100644 --- a/cmd/csaf_aggregator/mirror.go +++ b/cmd/csaf_aggregator/mirror.go @@ -451,23 +451,32 @@ func (w *worker) extractCategories(label string, advisory any) error { w.categories[label] = cats } - var result string - matcher := util.StringMatcher(&result) - const exprPrefix = "expr:" + var dynamic []string + matcher := util.StringTreeMatcher(&dynamic) + for _, cat := range categories { if strings.HasPrefix(cat, exprPrefix) { expr := cat[len(exprPrefix):] - if err := w.expr.Extract(expr, matcher, true, advisory); err != nil { - return err + // Compile first to check that the expression is okay. + if _, err := w.expr.Compile(expr); err != nil { + fmt.Printf("Compiling category expression %q failed: %v\n", + expr, err) + continue } - cats[result] = true + // Ignore errors here as they result from not matching. + w.expr.Extract(expr, matcher, true, advisory) } else { // Normal cats[cat] = true } } + // Add dynamic categories. + for _, cat := range dynamic { + cats[cat] = true + } + return nil } diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 635e061..74de519 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -19,6 +19,7 @@ import ( "html/template" "io" "log" + "net/http" "os" "github.com/csaf-poc/csaf_distribution/util" @@ -29,15 +30,16 @@ import ( var reportHTML string type options struct { - Output string `short:"o" long:"output" description:"File name of the generated report" value-name:"REPORT-FILE"` - Format string `short:"f" long:"format" choice:"json" choice:"html" description:"Format of report" default:"json"` - Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"` - ClientCert *string `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"` - ClientKey *string `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"` - Version bool `long:"version" description:"Display version of the binary"` - Verbose bool `long:"verbose" short:"v" description:"Verbose output"` - Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"` - Years *uint `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"` + Output string `short:"o" long:"output" description:"File name of the generated report" value-name:"REPORT-FILE"` + Format string `short:"f" long:"format" choice:"json" choice:"html" description:"Format of report" default:"json"` + Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"` + ClientCert *string `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"` + ClientKey *string `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"` + Version bool `long:"version" description:"Display version of the binary"` + Verbose bool `long:"verbose" short:"v" description:"Verbose output"` + Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"` + Years *uint `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"` + ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"` RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL"` RemoteValidatorCache string `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"` diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index b1895f0..e62c1ca 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -377,24 +377,30 @@ func (p *processor) httpClient() util.Client { TLSClientConfig: &tlsConfig, } - var client util.Client + client := util.Client(&hClient) + // Add extra headers. + if len(p.opts.ExtraHeader) > 0 { + client = &util.HeaderClient{ + Client: client, + Header: p.opts.ExtraHeader, + } + } + + // Add optional URL logging. if p.opts.Verbose { - client = &util.LoggingClient{Client: &hClient} - } else { - client = &hClient + client = &util.LoggingClient{Client: client} } - if p.opts.Rate == nil { - p.client = client - return client - } - - p.client = &util.LimitingClient{ - Client: client, - Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1), + // Add optional rate limiting. + if p.opts.Rate != nil { + client = &util.LimitingClient{ + Client: client, + Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1), + } } + p.client = client return p.client } diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 6674ef7..3bb2f93 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -64,24 +64,30 @@ func (d *downloader) httpClient() util.Client { } } - var client util.Client + client := util.Client(&hClient) + // Add extra headers. + if len(d.opts.ExtraHeader) > 0 { + client = &util.HeaderClient{ + Client: client, + Header: d.opts.ExtraHeader, + } + } + + // Add optional URL logging. if d.opts.Verbose { - client = &util.LoggingClient{Client: &hClient} - } else { - client = &hClient + client = &util.LoggingClient{Client: client} } - if d.opts.Rate == nil { - d.client = client - return client - } - - d.client = &util.LimitingClient{ - Client: client, - Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1), + // Add optional rate limiting. + if d.opts.Rate != nil { + client = &util.LimitingClient{ + Client: client, + Limiter: rate.NewLimiter(rate.Limit(*d.opts.Rate), 1), + } } + d.client = client return d.client } diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index 58ebe5e..38afe7d 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -12,6 +12,7 @@ package main import ( "fmt" "log" + "net/http" "os" "github.com/csaf-poc/csaf_distribution/util" @@ -19,11 +20,12 @@ import ( ) type options struct { - Directory *string `short:"d" long:"directory" description:"Directory to store the downloaded files in"` - Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"` - Version bool `long:"version" description:"Display version of the binary"` - Verbose bool `long:"verbose" short:"v" description:"Verbose output"` - Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"` + Directory *string `short:"d" long:"directory" description:"Directory to store the downloaded files in"` + Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"` + Version bool `long:"version" description:"Display version of the binary"` + Verbose bool `long:"verbose" short:"v" description:"Verbose output"` + Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"` + ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"` } func errCheck(err error) { diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index ea099e3..0fa4e5b 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -194,10 +194,17 @@ func (c *controller) upload(r *http.Request) (any, error) { // Check if we have to search for dynamic categories. var dynamicCategories []string if catExprs := c.cfg.DynamicCategories(); len(catExprs) > 0 { - var err error - if dynamicCategories, err = pe.Strings(catExprs, true, content); err != nil { - // XXX: Should we die here? - log.Printf("eval of dynamic catecory expressions failed: %v\n", err) + matcher := util.StringTreeMatcher(&dynamicCategories) + + for _, expr := range catExprs { + // Compile first to check that the expression is okay. + if _, err := pe.Compile(expr); err != nil { + log.Printf("Compiling category expression %q failed: %v\n", + expr, err) + continue + } + // Ignore errors here as they result from not matching. + pe.Extract(expr, matcher, true, content) } } diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index 949524f..6534840 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -19,13 +19,15 @@ Application Options: --validator=URL URL to validate documents remotely --validatorcache=FILE FILE to cache remote validations --validatorpreset= One or more presets to validate remotely (default: mandatory) + -H, --header= One or more extra HTTP header fields + Help Options: -h, --help Show this help message ``` Usage example: -` ./csaf_checker example.com -f html --rate=5.3 -o check-results.html` +` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html` Each performed check has a return type of either 0,1 or 2: ``` diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index c88a35b..56d54ce 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -4,8 +4,7 @@ A tool to download CSAF content from a specific domain/provider. ### Usage ``` -Usage: - csaf_downloader [OPTIONS] domain... +csaf_downloader [OPTIONS] domain... Application Options: -d, --directory= Directory to store the downloaded files in @@ -13,6 +12,7 @@ Application Options: --version Display version of the binary -v, --verbose Verbose output -r, --rate= The average upper limit of https operations per second + -H, --header= One or more extra HTTP header fields Help Options: -h, --help Show this help message diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index 81a45fa..d1e6b22 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -106,6 +106,18 @@ The following example file documents all available configuration options: # If a list item starts with `expr:` # the rest of the string is used as a JsonPath expression # to extract a string from the incoming advisories. +# If the result of the expression is a string this string +# is used. If the result is an array each element of +# this array is tested if it is a string or an array. +# If this test fails the expression fails. If the +# test succeeds the rules are applied recursively to +# collect all strings in the result. +# Suggested expressions are: +# - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name" +# - CVEs: "expr:$.vulnerabilities[*].cve" +# - CWEs: "expr:$.vulnerabilities[*].cwe.id" +# The used implementation to evaluate JSONPath expressions does +# not support the use of single-quotes. Double quotes have to be quoted. # Strings not starting with `expr:` are taken verbatim. # By default no category documents are created. # This example provides an overview over the syntax, diff --git a/docs/scripts/prepareUbuntuInstanceForITests.sh b/docs/scripts/prepareUbuntuInstanceForITests.sh index a062bbe..8811c53 100755 --- a/docs/scripts/prepareUbuntuInstanceForITests.sh +++ b/docs/scripts/prepareUbuntuInstanceForITests.sh @@ -11,4 +11,5 @@ apt install -y make bash sed tar git nginx fcgiwrap gnutls-bin # Install Go from binary distribution latest_go="$(curl https://go.dev/VERSION\?m=text).linux-amd64.tar.gz" curl -O https://dl.google.com/go/$latest_go +rm -rf /usr/local/go # be sure that we do not have an old installation tar -C /usr/local -xzf $latest_go diff --git a/util/client.go b/util/client.go index d6cd150..60a5bd0 100644 --- a/util/client.go +++ b/util/client.go @@ -14,6 +14,7 @@ import ( "log" "net/http" "net/url" + "strings" "golang.org/x/time/rate" ) @@ -38,6 +39,64 @@ type LimitingClient struct { Limiter *rate.Limiter } +// HeaderClient adds extra HTTP header fields to requests. +type HeaderClient struct { + Client + Header http.Header +} + +// Do implements the respective method of the [Client] interface. +func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) { + // Maybe this overly careful but this minimizes + // potential side effects in the caller. + orig := req.Header + defer func() { req.Header = orig }() + + // Work on a copy. + req.Header = req.Header.Clone() + + for key, values := range hc.Header { + for _, v := range values { + req.Header.Add(key, v) + } + } + return hc.Client.Do(req) +} + +// Get implements the respective method of the [Client] interface. +func (hc *HeaderClient) Get(url string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + return hc.Do(req) +} + +// Head implements the respective method of the [Client] interface. +func (hc *HeaderClient) Head(url string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodHead, url, nil) + if err != nil { + return nil, err + } + return hc.Do(req) +} + +// Post implements the respective method of the [Client] interface. +func (hc *HeaderClient) Post(url, contentType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) + return hc.Do(req) +} + +// PostForm implements the respective method of the [Client] interface. +func (hc *HeaderClient) PostForm(url string, data url.Values) (*http.Response, error) { + return hc.Post( + url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} + // Do implements the respective method of the Client interface. func (lc *LoggingClient) Do(req *http.Request) (*http.Response, error) { log.Printf("[DO]: %s\n", req.URL.String()) diff --git a/util/json.go b/util/json.go index ed0fd70..be9f330 100644 --- a/util/json.go +++ b/util/json.go @@ -42,6 +42,20 @@ func NewPathEval() *PathEval { } } +// Compile compiles an expression and stores it in the +// internal cache on success. +func (pe *PathEval) Compile(expr string) (gval.Evaluable, error) { + if eval := pe.exprs[expr]; eval != nil { + return eval, nil + } + eval, err := pe.builder.NewEvaluable(expr) + if err != nil { + return nil, err + } + pe.exprs[expr] = eval + return eval, nil +} + // Eval evalutes expression expr on document doc. // Returns the result of the expression. func (pe *PathEval) Eval(expr string, doc any) (any, error) { @@ -101,6 +115,37 @@ func StringMatcher(dst *string) func(any) error { } } +// StringTreeMatcher returns a matcher which adds strings +// to a slice and recursively strings from arrays of strings. +func StringTreeMatcher(strings *[]string) func(any) error { + // Only add unique strings. + unique := func(s string) { + for _, t := range *strings { + if s == t { + return + } + } + *strings = append(*strings, s) + } + var recurse func(any) error + recurse = func(x any) error { + switch y := x.(type) { + case string: + unique(y) + case []any: + for _, z := range y { + if err := recurse(z); err != nil { + return err + } + } + default: + return fmt.Errorf("unsupported type: %T", x) + } + return nil + } + return recurse +} + // TimeMatcher stores a time with a given format. func TimeMatcher(dst *time.Time, format string) func(any) error { return func(x any) error {