From aa574406cf3c679cecd6caf3ff1eb8b0c7d081ef Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 10 Mar 2023 10:39:23 +0100 Subject: [PATCH] Upgrade jsonschema library to 5.2.0 (#349) * make jsonschema loading work with current versions of the jsonschema library again and simplify the code while at it. * Improve itest workflow for validation service, to make them more robust. * Use a github action to setup nodejs and use a version that is required by https://github.com/secvisogram/csaf-validator-service/. * Add nodejs16 installation to prepareUbuntuInstanceForITests.sh. (so it can be done once in manual settings) * Add refreshing of apt cache before apt install because sometimes the cached ubuntu image does not have the apt cache current. --------- Co-authored-by: Bernhard Reiter --- .github/workflows/itest.yml | 6 ++ Makefile | 3 + csaf/validation.go | 93 ++++++++++--------- .../scripts/prepareUbuntuInstanceForITests.sh | 21 ++++- docs/scripts/setupValidationService.sh | 4 - go.mod | 2 +- go.sum | 6 +- 7 files changed, 78 insertions(+), 57 deletions(-) diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index 552e613..e3723a5 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -11,11 +11,17 @@ jobs: with: go-version: 1.19.1 + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: 16 + - name: Checkout uses: actions/checkout@v3 - name: Execute the scripts run: | + sudo apt update sudo apt install -y make nginx fcgiwrap gnutls-bin cp -r $GITHUB_WORKSPACE ~ cd ~ diff --git a/Makefile b/Makefile index 673d736..2e035e4 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,9 @@ tag_checked_out: # Note we need `--tags` because github release only creates lightweight tags # (see feature request https://github.com/github/feedback/discussions/4924). # We use `--always` in case of being run as github action with shallow clone. +# In this case we might in some situations see an error like +# `/bin/bash: line 1: 2b55bbb: value too great for base (error token is "2b55bbb")` +# which can be ignored. GITDESC := $(shell git describe --tags --always) GITDESCPATCH := $(shell echo '$(GITDESC)' | sed -E 's/v?[0-9]+\.[0-9]+\.([0-9]+)[-+]?.*/\1/') SEMVERPATCH := $(shell echo $$(( $(GITDESCPATCH) + 1 ))) diff --git a/csaf/validation.go b/csaf/validation.go index d4eb8e2..848b12b 100644 --- a/csaf/validation.go +++ b/csaf/validation.go @@ -11,6 +11,7 @@ package csaf import ( "bytes" _ "embed" // Used for embedding. + "io" "sort" "strings" "sync" @@ -39,62 +40,62 @@ var aggregatorSchema []byte //go:embed schema/ROLIE_feed_json_schema.json var rolieSchema []byte -var ( - compiledCSAFSchema compiledSchema - compiledProviderSchema compiledSchema - compiledAggregatorSchema compiledSchema - compiledRolieSchema compiledSchema -) - -func init() { - compiledCSAFSchema.compiler([]schemaData{ - {"https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", csafSchema}, - {"https://www.first.org/cvss/cvss-v2.0.json", cvss20}, - {"https://www.first.org/cvss/cvss-v3.0.json", cvss30}, - {"https://www.first.org/cvss/cvss-v3.1.json", cvss31}, - }) - compiledProviderSchema.compiler([]schemaData{ - {"https://docs.oasis-open.org/csaf/csaf/v2.0/provider_json_schema.json", providerSchema}, - {"https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", csafSchema}, - }) - compiledAggregatorSchema.compiler([]schemaData{ - {"https://docs.oasis-open.org/csaf/csaf/v2.0/aggregator_json_schema.json", aggregatorSchema}, - {"https://docs.oasis-open.org/csaf/csaf/v2.0/provider_json_schema.json", providerSchema}, - {"https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json", csafSchema}, - }) - compiledRolieSchema.compiler([]schemaData{ - {"https://raw.githubusercontent.com/tschmidtb51/csaf/ROLIE-schema/csaf_2.0/json_schema/ROLIE_feed_json_schema.json", rolieSchema}, - }) -} - -type schemaData struct { - url string - data []byte -} - type compiledSchema struct { + url string once sync.Once - compile func() err error compiled *jsonschema.Schema } -func (cs *compiledSchema) compiler(sds []schemaData) { - if len(sds) == 0 { - panic("missing schema data") +const ( + csafSchemaURL = "https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json" + providerSchemaURL = "https://docs.oasis-open.org/csaf/csaf/v2.0/provider_json_schema.json" + aggregatorSchemaURL = "https://docs.oasis-open.org/csaf/csaf/v2.0/aggregator_json_schema.json" + cvss20SchemaURL = "https://www.first.org/cvss/cvss-v2.0.json" + cvss30SchemaURL = "https://www.first.org/cvss/cvss-v3.0.json" + cvss31SchemaURL = "https://www.first.org/cvss/cvss-v3.1.json" + rolieSchemaURL = "https://raw.githubusercontent.com/tschmidtb51/csaf/ROLIE-schema/csaf_2.0/json_schema/ROLIE_feed_json_schema.json" +) + +var ( + compiledCSAFSchema = compiledSchema{url: csafSchemaURL} + compiledProviderSchema = compiledSchema{url: providerSchemaURL} + compiledAggregatorSchema = compiledSchema{url: aggregatorSchemaURL} + compiledRolieSchema = compiledSchema{url: rolieSchemaURL} +) + +// loadURL loads the content of an URL from embedded data or +// falls back to the global loader function of the jsonschema package. +func loadURL(s string) (io.ReadCloser, error) { + loader := func(data []byte) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil } - cs.compile = func() { - c := jsonschema.NewCompiler() - for _, s := range sds { - if cs.err = c.AddResource( - s.url, bytes.NewReader(s.data)); cs.err != nil { - return - } - } - cs.compiled, cs.err = c.Compile(sds[0].url) + switch s { + case csafSchemaURL: + return loader(csafSchema) + case cvss20SchemaURL: + return loader(cvss20) + case cvss30SchemaURL: + return loader(cvss30) + case cvss31SchemaURL: + return loader(cvss31) + case providerSchemaURL: + return loader(providerSchema) + case aggregatorSchemaURL: + return loader(aggregatorSchema) + case rolieSchemaURL: + return loader(rolieSchema) + default: + return jsonschema.LoadURL(s) } } +func (cs *compiledSchema) compile() { + c := jsonschema.NewCompiler() + c.LoadURL = loadURL + cs.compiled, cs.err = c.Compile(cs.url) +} + func (cs *compiledSchema) validate(doc any) ([]string, error) { cs.once.Do(cs.compile) diff --git a/docs/scripts/prepareUbuntuInstanceForITests.sh b/docs/scripts/prepareUbuntuInstanceForITests.sh index 8811c53..35a1745 100755 --- a/docs/scripts/prepareUbuntuInstanceForITests.sh +++ b/docs/scripts/prepareUbuntuInstanceForITests.sh @@ -5,11 +5,28 @@ set -e # for the csaf_distribution integration tests # by installing the required packages. - -apt install -y make bash sed tar git nginx fcgiwrap gnutls-bin +apt update +apt install -y make bash curl gnupg sed tar git nginx fcgiwrap gnutls-bin # Install Go from binary distribution latest_go="$(curl https://go.dev/VERSION\?m=text).linux-amd64.tar.gz" curl -O https://dl.google.com/go/$latest_go rm -rf /usr/local/go # be sure that we do not have an old installation tar -C /usr/local -xzf $latest_go + +# Install newer Node.js version from nodesource +# as needed for https://github.com/secvisogram/csaf-validator-service +# Instructions from +# https://github.com/nodesource/distributions/blob/master/README.md#debmanual +KEYRING=/usr/share/keyrings/nodesource.gpg +curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > "$KEYRING" +gpg --no-default-keyring --keyring "$KEYRING" --list-keys +chmod a+r /usr/share/keyrings/nodesource.gpg + +VERSION=node_16.x +DISTRO="$(lsb_release -s -c)" +echo "deb [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | tee /etc/apt/sources.list.d/nodesource.list +echo "deb-src [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | tee -a /etc/apt/sources.list.d/nodesource.list + +apt-get update +apt-get install -y nodejs diff --git a/docs/scripts/setupValidationService.sh b/docs/scripts/setupValidationService.sh index bc7629d..a0e4f6e 100755 --- a/docs/scripts/setupValidationService.sh +++ b/docs/scripts/setupValidationService.sh @@ -21,10 +21,6 @@ echo ' remote_validator= { "url" = "http://localhost:8082", "presets" = ["mandatory"], "cache" = "/var/lib/csaf/validations.db" } ' | sudo tee --append /etc/csaf/config.toml -# Install nodejs -curl -fsSL https://deb.nodesource.com/setup_14.x | sudo -E bash - -sudo apt-get install -y nodejs - npm install pm2 -g pushd ~ diff --git a/go.mod b/go.mod index 46716f8..ebb9efd 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gofrs/flock v0.8.1 github.com/jessevdk/go-flags v1.5.0 github.com/mitchellh/go-homedir v1.1.0 - github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 go.etcd.io/bbolt v1.3.7 golang.org/x/crypto v0.6.0 golang.org/x/term v0.5.0 diff --git a/go.sum b/go.sum index f8d9df1..9b01983 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE= -github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 h1:WCcC4vZDS1tYNxjWlwRJZQy28r8CMoggKnxNzxsVDMQ= +github.com/santhosh-tekuri/jsonschema/v5 v5.2.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -46,8 +46,6 @@ go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=