Upgrade jsonschema library to 5.2.0 (#349)

* make jsonschema loading work with current versions of the jsonschema library
   again and simplify the code while at it.
* Improve itest workflow for validation service, to make them more robust.
    * Use a github action to setup nodejs and use a version that is
       required by https://github.com/secvisogram/csaf-validator-service/.
    * Add nodejs16 installation to prepareUbuntuInstanceForITests.sh.
       (so it can be done once in manual settings)
    * Add refreshing of apt cache before apt install because sometimes
       the cached ubuntu image does not have the apt cache current.

---------

Co-authored-by: Bernhard Reiter <bernhard@intevation.de>

.github/workflows/itest.yml

@@ -11,11 +11,17 @@ jobs:
         with:
           go-version: 1.19.1
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: Execute the scripts
         run: |
+          sudo apt update
           sudo apt install -y make nginx fcgiwrap gnutls-bin
           cp -r $GITHUB_WORKSPACE ~
           cd ~

Makefile

@@ -44,6 +44,9 @@ tag_checked_out:
 # Note we need `--tags` because github release only creates lightweight tags
 #   (see feature request https://github.com/github/feedback/discussions/4924).
 #   We use `--always` in case of being run as github action with shallow clone.
+#   In this case we might in some situations see an error like
+#   `/bin/bash: line 1: 2b55bbb: value too great for base (error token is "2b55bbb")`
+#   which can be ignored.
 GITDESC := $(shell git describe --tags --always)
 GITDESCPATCH := $(shell echo '$(GITDESC)' | sed -E 's/v?[0-9]+\.[0-9]+\.([0-9]+)[-+]?.*/\1/')
 SEMVERPATCH := $(shell echo $$(( $(GITDESCPATCH) + 1 )))

csaf/validation.go

@@ -11,6 +11,7 @@ package csaf
 import (
 	"bytes"
 	_ "embed" // Used for embedding.
+	"io"
 	"sort"
 	"strings"
 	"sync"
@@ -39,62 +40,62 @@ var aggregatorSchema []byte
 //go:embed schema/ROLIE_feed_json_schema.json
 var rolieSchema []byte
 
-var (
-	compiledCSAFSchema       compiledSchema
-	compiledProviderSchema   compiledSchema
-	compiledAggregatorSchema compiledSchema
-	compiledRolieSchema      compiledSchema
-)
-
-func init() {
-	compiledCSAFSchema.compiler([]schemaData{
-		{"https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", csafSchema},
-		{"https://www.first.org/cvss/cvss-v2.0.json", cvss20},
-		{"https://www.first.org/cvss/cvss-v3.0.json", cvss30},
-		{"https://www.first.org/cvss/cvss-v3.1.json", cvss31},
-	})
-	compiledProviderSchema.compiler([]schemaData{
-		{"https://docs.oasis-open.org/csaf/csaf/v2.0/provider_json_schema.json", providerSchema},
-		{"https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", csafSchema},
-	})
-	compiledAggregatorSchema.compiler([]schemaData{
-		{"https://docs.oasis-open.org/csaf/csaf/v2.0/aggregator_json_schema.json", aggregatorSchema},
-		{"https://docs.oasis-open.org/csaf/csaf/v2.0/provider_json_schema.json", providerSchema},
-		{"https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", csafSchema},
-	})
-	compiledRolieSchema.compiler([]schemaData{
-		{"https://raw.githubusercontent.com/tschmidtb51/csaf/ROLIE-schema/csaf_2.0/json_schema/ROLIE_feed_json_schema.json", rolieSchema},
-	})
-}
-
-type schemaData struct {
-	url  string
-	data []byte
-}
-
 type compiledSchema struct {
+	url      string
 	once     sync.Once
-	compile  func()
 	err      error
 	compiled *jsonschema.Schema
 }
 
-func (cs *compiledSchema) compiler(sds []schemaData) {
+const (
-	if len(sds) == 0 {
+	csafSchemaURL       = "https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json"
-		panic("missing schema data")
+	providerSchemaURL   = "https://docs.oasis-open.org/csaf/csaf/v2.0/provider_json_schema.json"
+	aggregatorSchemaURL = "https://docs.oasis-open.org/csaf/csaf/v2.0/aggregator_json_schema.json"
+	cvss20SchemaURL     = "https://www.first.org/cvss/cvss-v2.0.json"
+	cvss30SchemaURL     = "https://www.first.org/cvss/cvss-v3.0.json"
+	cvss31SchemaURL     = "https://www.first.org/cvss/cvss-v3.1.json"
+	rolieSchemaURL      = "https://raw.githubusercontent.com/tschmidtb51/csaf/ROLIE-schema/csaf_2.0/json_schema/ROLIE_feed_json_schema.json"
+)
+
+var (
+	compiledCSAFSchema       = compiledSchema{url: csafSchemaURL}
+	compiledProviderSchema   = compiledSchema{url: providerSchemaURL}
+	compiledAggregatorSchema = compiledSchema{url: aggregatorSchemaURL}
+	compiledRolieSchema      = compiledSchema{url: rolieSchemaURL}
+)
+
+// loadURL loads the content of an URL from embedded data or
+// falls back to the global loader function of the jsonschema package.
+func loadURL(s string) (io.ReadCloser, error) {
+	loader := func(data []byte) (io.ReadCloser, error) {
+		return io.NopCloser(bytes.NewReader(data)), nil
 	}
-	cs.compile = func() {
+	switch s {
-		c := jsonschema.NewCompiler()
+	case csafSchemaURL:
-		for _, s := range sds {
+		return loader(csafSchema)
-			if cs.err = c.AddResource(
+	case cvss20SchemaURL:
-				s.url, bytes.NewReader(s.data)); cs.err != nil {
+		return loader(cvss20)
-				return
+	case cvss30SchemaURL:
-			}
+		return loader(cvss30)
-		}
+	case cvss31SchemaURL:
-		cs.compiled, cs.err = c.Compile(sds[0].url)
+		return loader(cvss31)
+	case providerSchemaURL:
+		return loader(providerSchema)
+	case aggregatorSchemaURL:
+		return loader(aggregatorSchema)
+	case rolieSchemaURL:
+		return loader(rolieSchema)
+	default:
+		return jsonschema.LoadURL(s)
 	}
 }
 
+func (cs *compiledSchema) compile() {
+	c := jsonschema.NewCompiler()
+	c.LoadURL = loadURL
+	cs.compiled, cs.err = c.Compile(cs.url)
+}
+
 func (cs *compiledSchema) validate(doc any) ([]string, error) {
 	cs.once.Do(cs.compile)
 

docs/scripts/prepareUbuntuInstanceForITests.sh

@@ -5,11 +5,28 @@ set -e
 # for the csaf_distribution integration tests
 # by installing the required packages.
 
-
+apt update
-apt install -y make bash sed tar git nginx fcgiwrap gnutls-bin
+apt install -y make bash curl gnupg sed tar git nginx fcgiwrap gnutls-bin
 
 # Install Go from binary distribution
 latest_go="$(curl https://go.dev/VERSION\?m=text).linux-amd64.tar.gz"
 curl -O https://dl.google.com/go/$latest_go
 rm -rf /usr/local/go # be sure that we do not have an old installation
 tar -C /usr/local -xzf $latest_go
+
+# Install newer Node.js version from nodesource
+# as needed for https://github.com/secvisogram/csaf-validator-service
+# Instructions from
+#  https://github.com/nodesource/distributions/blob/master/README.md#debmanual
+KEYRING=/usr/share/keyrings/nodesource.gpg
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > "$KEYRING"
+gpg --no-default-keyring --keyring "$KEYRING" --list-keys
+chmod a+r /usr/share/keyrings/nodesource.gpg
+
+VERSION=node_16.x
+DISTRO="$(lsb_release -s -c)"
+echo "deb [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | tee /etc/apt/sources.list.d/nodesource.list
+echo "deb-src [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | tee -a /etc/apt/sources.list.d/nodesource.list
+
+apt-get update
+apt-get install -y nodejs

docs/scripts/setupValidationService.sh

@@ -21,10 +21,6 @@ echo '
 remote_validator= { "url" = "http://localhost:8082", "presets" = ["mandatory"], "cache" = "/var/lib/csaf/validations.db" }
 ' | sudo tee --append /etc/csaf/config.toml
 
-# Install nodejs
-curl -fsSL https://deb.nodesource.com/setup_14.x | sudo -E bash -
-sudo apt-get install -y nodejs
-
 npm install pm2 -g
 
 pushd ~

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/gofrs/flock v0.8.1
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0
+	github.com/santhosh-tekuri/jsonschema/v5 v5.2.0
 	go.etcd.io/bbolt v1.3.7
 	golang.org/x/crypto v0.6.0
 	golang.org/x/term v0.5.0

go.sum

@@ -33,8 +33,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE=
+github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 h1:WCcC4vZDS1tYNxjWlwRJZQy28r8CMoggKnxNzxsVDMQ=
-github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
+github.com/santhosh-tekuri/jsonschema/v5 v5.2.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -46,8 +46,6 @@ go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=