Add aggregator; improve itest workflow

* Factor JSON evaluation and  construction base URLs out of of checker.
* Move json path matching to util.
* Add csaf_aggregator (as additional command)
* Improve itest workflow to checkout the branch where it is running on.

resolve #105
resolve  #72

Co-authored-by: tschmidtb51 <65305130+tschmidtb51@users.noreply.github.com>
Co-authored-by: Bernhard Reiter <bernhard@intevation.de>
Co-authored-by: Fadi Abbud <fadi.abbud@intevation.de>

util/file.go

@@ -13,10 +13,35 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
+	"strings"
 	"time"
 )
 
+var (
+	twoOrMoreDots = regexp.MustCompile(`\.{2,}`)
+	stripSlashes  = strings.NewReplacer(`/`, ``, `\`, ``)
+)
+
+// CleanFileName removes the "/" "\" charachters and replace the two or more
+// occurences of "." with only one from the passed string.
+func CleanFileName(s string) string {
+	return twoOrMoreDots.ReplaceAllString(stripSlashes.Replace(s), `.`)
+}
+
+// PathExists returns true if path exits.
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		err = nil
+	}
+	return false, err
+}
+
 // NWriter is an io.Writer counting the bytes copied through it.
 type NWriter struct {
 	io.Writer

util/hash.go

@@ -0,0 +1,66 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
+
+package util
+
+import (
+	"bufio"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+	"regexp"
+)
+
+var hexRe = regexp.MustCompile(`^([[:xdigit:]]+)`)
+
+// HashFromReader reads a base 16 coded hash sum from a reader.
+func HashFromReader(r io.Reader) ([]byte, error) {
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		if m := hexRe.FindStringSubmatch(scanner.Text()); m != nil {
+			return hex.DecodeString(m[1])
+		}
+	}
+	return nil, scanner.Err()
+}
+
+// HashFromFile reads a base 16 coded hash sum from a file.
+func HashFromFile(fname string) ([]byte, error) {
+	f, err := os.Open(fname)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return HashFromReader(f)
+}
+
+// WriteHashToFile writes a hash of data to file fname.
+func WriteHashToFile(fname, name string, h hash.Hash, data []byte) error {
+	if _, err := h.Write(data); err != nil {
+		return err
+	}
+
+	f, err := os.Create(fname)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(f, "%x %s\n", h.Sum(nil), name)
+	return f.Close()
+}
+
+// WriteHashSumToFile writes a hash sum to file fname.
+func WriteHashSumToFile(fname, name string, sum []byte) error {
+	f, err := os.Create(fname)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(f, "%x %s\n", sum, name)
+	return f.Close()
+}

util/json.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"time"
 
 	"github.com/PaesslerAG/gval"
@@ -76,6 +77,18 @@ func ReMarshalMatcher(dst interface{}) func(interface{}) error {
 	}
 }
 
+// BoolMatcher stores the matched result in a bool.
+func BoolMatcher(dst *bool) func(interface{}) error {
+	return func(x interface{}) error {
+		b, ok := x.(bool)
+		if !ok {
+			return errors.New("not a bool")
+		}
+		*dst = b
+		return nil
+	}
+}
+
 // StringMatcher stores the matched result in a string.
 func StringMatcher(dst *string) func(interface{}) error {
 	return func(x interface{}) error {
@@ -111,14 +124,17 @@ func (pe *PathEval) Extract(
 	optional bool,
 	doc interface{},
 ) error {
-	x, err := pe.Eval(expr, doc)
-	if err != nil {
-		if optional {
+	optErr := func(err error) error {
+		if err == nil || optional {
 			return nil
 		}
-		return err
+		return fmt.Errorf("extract failed '%s': %v", expr, err)
 	}
-	return action(x)
+	x, err := pe.Eval(expr, doc)
+	if err != nil {
+		return optErr(err)
+	}
+	return optErr(action(x))
 }
 
 // Match matches a list of PathEvalMatcher pairs against a document.

util/url.go

@@ -0,0 +1,34 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
+
+package util
+
+import (
+	"net/url"
+	"strings"
+)
+
+// BaseURL returns the base URL for a given URL p.
+func BaseURL(p string) (string, error) {
+	u, err := url.Parse(p)
+	if err != nil {
+		return "", err
+	}
+	ep := u.EscapedPath()
+	if idx := strings.LastIndexByte(ep, '/'); idx != -1 {
+		ep = ep[:idx+1]
+	}
+	user := u.User.String()
+	if user != "" {
+		user += "@"
+	}
+	if !strings.HasPrefix(ep, "/") {
+		ep = "/" + ep
+	}
+	return u.Scheme + "://" + user + u.Host + ep, nil
+}