From 8c8ccf6a2e7dbd746b73c0e564eb3fafc432e58c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 22 Jan 2023 20:55:26 +0100 Subject: [PATCH 1/8] Extract more than one string from expr: category fields. --- cmd/csaf_aggregator/mirror.go | 9 ++++--- cmd/csaf_provider/actions.go | 5 ++-- docs/csaf_provider.md | 6 +++++ util/json.go | 50 +++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go index 7fe71d7..3ea4837 100644 --- a/cmd/csaf_aggregator/mirror.go +++ b/cmd/csaf_aggregator/mirror.go @@ -451,18 +451,19 @@ func (w *worker) extractCategories(label string, advisory any) error { w.categories[label] = cats } - var result string - matcher := util.StringMatcher(&result) - const exprPrefix = "expr:" for _, cat := range categories { if strings.HasPrefix(cat, exprPrefix) { expr := cat[len(exprPrefix):] + var results []string + matcher := util.StringTreeMatcher(&results) if err := w.expr.Extract(expr, matcher, true, advisory); err != nil { return err } - cats[result] = true + for _, result := range results { + cats[result] = true + } } else { // Normal cats[cat] = true } diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index ea099e3..b1ad0a9 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -195,9 +195,10 @@ func (c *controller) upload(r *http.Request) (any, error) { var dynamicCategories []string if catExprs := c.cfg.DynamicCategories(); len(catExprs) > 0 { var err error - if dynamicCategories, err = pe.Strings(catExprs, true, content); err != nil { + if dynamicCategories, err = pe.StringsFromTree( + catExprs, true, content); err != nil { // XXX: Should we die here? - log.Printf("eval of dynamic catecory expressions failed: %v\n", err) + log.Printf("eval of dynamic catergory expressions failed: %v\n", err) } } diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index 81a45fa..464f0db 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -106,6 +106,12 @@ The following example file documents all available configuration options: # If a list item starts with `expr:` # the rest of the string is used as a JsonPath expression # to extract a string from the incoming advisories. +# If the result of the expression is a string this string +# is used. If the result is an array each element of +# this array is tested if it is a string or an array. +# If this test fails the expression fails. If the +# test succeeds the rules are applied recursively to +# collect all strings in the result. # Strings not starting with `expr:` are taken verbatim. # By default no category documents are created. # This example provides an overview over the syntax, diff --git a/util/json.go b/util/json.go index ed0fd70..48d0e09 100644 --- a/util/json.go +++ b/util/json.go @@ -101,6 +101,37 @@ func StringMatcher(dst *string) func(any) error { } } +// StringTreeMatcher returns a matcher which add strings from +// stringss and recursively from arrays from strings. +func StringTreeMatcher(strings *[]string) func(any) error { + // Only add unique strings. + unique := func(s string) { + for _, t := range *strings { + if s == t { + return + } + } + *strings = append(*strings, s) + } + var recurse func(any) error + recurse = func(x any) error { + switch y := x.(type) { + case string: + unique(y) + case []any: + for _, z := range y { + if err := recurse(z); err != nil { + return err + } + } + default: + return fmt.Errorf("unsupported type: %T", x) + } + return nil + } + return recurse +} + // TimeMatcher stores a time with a given format. func TimeMatcher(dst *time.Time, format string) func(any) error { return func(x any) error { @@ -147,6 +178,25 @@ func (pe *PathEval) Match(matcher []PathEvalMatcher, doc any) error { return nil } +// StringsFromTree returns strings from the given exprs. +// 1. If a expression results a string this string is used. +// 2. if a expression results in an array the elements +// of this array are recursively treated with 1. and 2. +func (pe *PathEval) StringsFromTree( + exprs []string, + optional bool, + doc any, +) ([]string, error) { + results := make([]string, 0, len(exprs)) + matcher := StringTreeMatcher(&results) + for _, expr := range exprs { + if err := pe.Extract(expr, matcher, optional, doc); err != nil { + return nil, err + } + } + return results, nil +} + // Strings searches the given document for the given set of expressions // and returns the corresponding strings. The optional flag indicates // if the expression evaluation have to succseed or not. From 6bf8c530c6f8f2b02c5b58121c262a4fdb5c6498 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 23 Jan 2023 14:30:47 +0100 Subject: [PATCH 2/8] Fix comment typo --- util/json.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/json.go b/util/json.go index 48d0e09..d21b4cf 100644 --- a/util/json.go +++ b/util/json.go @@ -101,8 +101,8 @@ func StringMatcher(dst *string) func(any) error { } } -// StringTreeMatcher returns a matcher which add strings from -// stringss and recursively from arrays from strings. +// StringTreeMatcher returns a matcher which addis strings from +// strings and recursively from arrays of strings. func StringTreeMatcher(strings *[]string) func(any) error { // Only add unique strings. unique := func(s string) { From bf6dfafffdf79f2697e7a00912fe09e337b8b3ee Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 24 Jan 2023 01:34:23 +0100 Subject: [PATCH 3/8] Address remarks from review. --- docs/csaf_provider.md | 10 ++++++++++ util/json.go | 8 ++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index 464f0db..4db4d31 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -112,6 +112,16 @@ The following example file documents all available configuration options: # If this test fails the expression fails. If the # test succeeds the rules are applied recursively to # collect all strings in the result. +# If the result of the expression is a string this string +# is used. If the result is an array each element of +# this array is tested if it is a string or an array. +# If this test fails the expression fails. If the +# test succeeds the rules are applied recursively to +# collect all strings in the result. +# Suggested expressions are: +# - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category=='vendor' || @.category=='product_family' || @.category=='product_name')].name" +# - CVEs: "expr:$.vulnerabilities[*].cve" +# - CWEs: "expr:$.vulnerabilities[*].cwe.id" # Strings not starting with `expr:` are taken verbatim. # By default no category documents are created. # This example provides an overview over the syntax, diff --git a/util/json.go b/util/json.go index d21b4cf..985c7e1 100644 --- a/util/json.go +++ b/util/json.go @@ -101,8 +101,8 @@ func StringMatcher(dst *string) func(any) error { } } -// StringTreeMatcher returns a matcher which addis strings from -// strings and recursively from arrays of strings. +// StringTreeMatcher returns a matcher which adds strings +// to a slice and recursively strings from arrays of strings. func StringTreeMatcher(strings *[]string) func(any) error { // Only add unique strings. unique := func(s string) { @@ -179,8 +179,8 @@ func (pe *PathEval) Match(matcher []PathEvalMatcher, doc any) error { } // StringsFromTree returns strings from the given exprs. -// 1. If a expression results a string this string is used. -// 2. if a expression results in an array the elements +// 1. If an expression results in a string this string is used. +// 2. if an expression results in an array the elements // of this array are recursively treated with 1. and 2. func (pe *PathEval) StringsFromTree( exprs []string, From 6dedeff7fc0b8635f95f6440ab7e0b19d53c128f Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer <107021473+JanHoefelmeyer@users.noreply.github.com> Date: Wed, 25 Jan 2023 11:40:25 +0100 Subject: [PATCH 4/8] Update actions.go Fixes typo: catergory -> category --- cmd/csaf_provider/actions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index b1ad0a9..67b36a5 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -198,7 +198,7 @@ func (c *controller) upload(r *http.Request) (any, error) { if dynamicCategories, err = pe.StringsFromTree( catExprs, true, content); err != nil { // XXX: Should we die here? - log.Printf("eval of dynamic catergory expressions failed: %v\n", err) + log.Printf("eval of dynamic category expressions failed: %v\n", err) } } From 69dda45bac1ae7ae4733a3b4c0e9caa8d160cb51 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 25 Jan 2023 13:42:26 +0100 Subject: [PATCH 5/8] Dedup doc --- docs/csaf_provider.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index 4db4d31..177e71d 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -112,12 +112,6 @@ The following example file documents all available configuration options: # If this test fails the expression fails. If the # test succeeds the rules are applied recursively to # collect all strings in the result. -# If the result of the expression is a string this string -# is used. If the result is an array each element of -# this array is tested if it is a string or an array. -# If this test fails the expression fails. If the -# test succeeds the rules are applied recursively to -# collect all strings in the result. # Suggested expressions are: # - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category=='vendor' || @.category=='product_family' || @.category=='product_name')].name" # - CVEs: "expr:$.vulnerabilities[*].cve" From ad5c678abf7d4faa164b827fffcdfdb79bea52ae Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 26 Jan 2023 18:55:21 +0100 Subject: [PATCH 6/8] Mention the single quote limitation of the jsonpath implementation --- docs/csaf_provider.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index 177e71d..c0b8f3d 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -113,9 +113,10 @@ The following example file documents all available configuration options: # test succeeds the rules are applied recursively to # collect all strings in the result. # Suggested expressions are: -# - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category=='vendor' || @.category=='product_family' || @.category=='product_name')].name" +# - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name" # - CVEs: "expr:$.vulnerabilities[*].cve" # - CWEs: "expr:$.vulnerabilities[*].cwe.id" +# s implementation of JsonPath expressions does not support the use of single-quotes. Double quotes have to be quoted. # Strings not starting with `expr:` are taken verbatim. # By default no category documents are created. # This example provides an overview over the syntax, From ff31ebfa0fac198305788b90e77822281258695c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 26 Jan 2023 20:02:05 +0100 Subject: [PATCH 7/8] Fixed typos --- docs/csaf_provider.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index c0b8f3d..d1e6b22 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -116,7 +116,8 @@ The following example file documents all available configuration options: # - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name" # - CVEs: "expr:$.vulnerabilities[*].cve" # - CWEs: "expr:$.vulnerabilities[*].cwe.id" -# s implementation of JsonPath expressions does not support the use of single-quotes. Double quotes have to be quoted. +# The used implementation to evaluate JSONPath expressions does +# not support the use of single-quotes. Double quotes have to be quoted. # Strings not starting with `expr:` are taken verbatim. # By default no category documents are created. # This example provides an overview over the syntax, From 0745a0943d4050e864f00f9fc83a075d32d51d60 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 26 Jan 2023 21:54:46 +0100 Subject: [PATCH 8/8] Separate compiling and evaluation of dynamic categories. --- cmd/csaf_aggregator/mirror.go | 22 +++++++++++++++------- cmd/csaf_provider/actions.go | 16 +++++++++++----- util/json.go | 33 ++++++++++++++------------------- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go index 3ea4837..fd37b3e 100644 --- a/cmd/csaf_aggregator/mirror.go +++ b/cmd/csaf_aggregator/mirror.go @@ -453,22 +453,30 @@ func (w *worker) extractCategories(label string, advisory any) error { const exprPrefix = "expr:" + var dynamic []string + matcher := util.StringTreeMatcher(&dynamic) + for _, cat := range categories { if strings.HasPrefix(cat, exprPrefix) { expr := cat[len(exprPrefix):] - var results []string - matcher := util.StringTreeMatcher(&results) - if err := w.expr.Extract(expr, matcher, true, advisory); err != nil { - return err - } - for _, result := range results { - cats[result] = true + // Compile first to check that the expression is okay. + if _, err := w.expr.Compile(expr); err != nil { + fmt.Printf("Compiling category expression %q failed: %v\n", + expr, err) + continue } + // Ignore errors here as they result from not matching. + w.expr.Extract(expr, matcher, true, advisory) } else { // Normal cats[cat] = true } } + // Add dynamic categories. + for _, cat := range dynamic { + cats[cat] = true + } + return nil } diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index 67b36a5..0fa4e5b 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -194,11 +194,17 @@ func (c *controller) upload(r *http.Request) (any, error) { // Check if we have to search for dynamic categories. var dynamicCategories []string if catExprs := c.cfg.DynamicCategories(); len(catExprs) > 0 { - var err error - if dynamicCategories, err = pe.StringsFromTree( - catExprs, true, content); err != nil { - // XXX: Should we die here? - log.Printf("eval of dynamic category expressions failed: %v\n", err) + matcher := util.StringTreeMatcher(&dynamicCategories) + + for _, expr := range catExprs { + // Compile first to check that the expression is okay. + if _, err := pe.Compile(expr); err != nil { + log.Printf("Compiling category expression %q failed: %v\n", + expr, err) + continue + } + // Ignore errors here as they result from not matching. + pe.Extract(expr, matcher, true, content) } } diff --git a/util/json.go b/util/json.go index 985c7e1..be9f330 100644 --- a/util/json.go +++ b/util/json.go @@ -42,6 +42,20 @@ func NewPathEval() *PathEval { } } +// Compile compiles an expression and stores it in the +// internal cache on success. +func (pe *PathEval) Compile(expr string) (gval.Evaluable, error) { + if eval := pe.exprs[expr]; eval != nil { + return eval, nil + } + eval, err := pe.builder.NewEvaluable(expr) + if err != nil { + return nil, err + } + pe.exprs[expr] = eval + return eval, nil +} + // Eval evalutes expression expr on document doc. // Returns the result of the expression. func (pe *PathEval) Eval(expr string, doc any) (any, error) { @@ -178,25 +192,6 @@ func (pe *PathEval) Match(matcher []PathEvalMatcher, doc any) error { return nil } -// StringsFromTree returns strings from the given exprs. -// 1. If an expression results in a string this string is used. -// 2. if an expression results in an array the elements -// of this array are recursively treated with 1. and 2. -func (pe *PathEval) StringsFromTree( - exprs []string, - optional bool, - doc any, -) ([]string, error) { - results := make([]string, 0, len(exprs)) - matcher := StringTreeMatcher(&results) - for _, expr := range exprs { - if err := pe.Extract(expr, matcher, optional, doc); err != nil { - return nil, err - } - } - return results, nil -} - // Strings searches the given document for the given set of expressions // and returns the corresponding strings. The optional flag indicates // if the expression evaluation have to succseed or not.