Remote validator output (#347)

* The validator is now able to print the details of the remote validations.
---------

Co-authored-by: JanHoefelmeyer <hoefelmeyer.jan@gmail.com>
Co-authored-by: JanHoefelmeyer <Jan Höfelmeyer jhoefelmeyer@intevation.de>
Co-authored-by: Sascha L. Teichmann <sascha.teichmann@intevation.de>

cmd/csaf_aggregator/mirror.go

@@ -539,12 +539,12 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 
 		// Check against remote validator.
 		if rmv := w.processor.remoteValidator; rmv != nil {
-			valid, err := rmv.Validate(advisory)
+			rvr, err := rmv.Validate(advisory)
 			if err != nil {
 				log.Printf("Calling remote validator failed: %s\n", err)
 				continue
 			}
-			if !valid {
+			if !rvr.Valid {
 				log.Printf(
 					"CSAF file %s does not validate remotely.\n", file)
 				continue

cmd/csaf_checker/processor.go

@@ -514,9 +514,9 @@ func (p *processor) integrity(
 
 		// Validate against remote validator.
 		if p.validator != nil {
-			if ok, err := p.validator.Validate(doc); err != nil {
+			if rvr, err := p.validator.Validate(doc); err != nil {
 				p.invalidAdvisories.error("Calling remote validator on %s failed: %v", u, err)
-			} else if !ok {
+			} else if !rvr.Valid {
 				p.invalidAdvisories.error("Remote validation of %s failed.", u)
 			}
 		}

cmd/csaf_downloader/downloader.go

@@ -370,13 +370,13 @@ func (d *downloader) downloadFiles(label csaf.TLPLabel, files []csaf.AdvisoryFil
 
 		// Validate against remote validator
 		if d.validator != nil {
-			ok, err := d.validator.Validate(doc)
+			rvr, err := d.validator.Validate(doc)
 			if err != nil {
 				return fmt.Errorf(
 					"calling remote validator on %q failed: %w",
 					file.URL(), err)
 			}
-			if !ok {
+			if !rvr.Valid {
 				log.Printf("Remote validation of %q failed\n", file.URL())
 			}
 		}

cmd/csaf_provider/actions.go

@@ -179,11 +179,11 @@ func (c *controller) upload(r *http.Request) (any, error) {
 		if err != nil {
 			return nil, err
 		}
-		valid, err := validator.Validate(content)
+		rvr, err := validator.Validate(content)
 		if err != nil {
 			return nil, err
 		}
-		if !valid {
+		if !rvr.Valid {
 			return nil, errors.New("does not validate against remote validator")
 		}
 	}

cmd/csaf_validator/main.go

@@ -26,6 +26,7 @@ type options struct {
 	RemoteValidator        string   `long:"validator" description:"URL to validate documents remotely" value-name:"URL"`
 	RemoteValidatorCache   string   `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE"`
 	RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more presets to validate remotely" default:"mandatory"`
+	Output                 string   `short:"o" long:"output" description:"If a remote validator was used, display AMOUNT ('all', 'important' or 'short') results" value-name:"AMOUNT"`
 }
 
 func main() {
@@ -68,6 +69,21 @@ func run(opts *options, files []string) error {
 		defer validator.Close()
 	}
 
+	// Select amount level of output for remote validation.
+	var printResult func(*csaf.RemoteValidationResult)
+	switch opts.Output {
+	case "all":
+		printResult = printAll
+	case "short":
+		printResult = printShort
+	case "important":
+		printResult = printImportant
+	case "":
+		printResult = noPrint
+	default:
+		return fmt.Errorf("unknown output amount %q", opts.Output)
+	}
+
 	for _, file := range files {
 		// Check if the file name is valid.
 		if !util.ConformingFileName(filepath.Base(file)) {
@@ -95,13 +111,14 @@ func run(opts *options, files []string) error {
 		}
 		// Validate against remote validator.
 		if validator != nil {
-			validate, err := validator.Validate(doc)
+			rvr, err := validator.Validate(doc)
 			if err != nil {
 				return fmt.Errorf("remote validation of %q failed: %w",
 					file, err)
 			}
+			printResult(rvr)
 			var passes string
-			if validate {
+			if rvr.Valid {
 				passes = "passes"
 			} else {
 				passes = "does not pass"
@@ -113,6 +130,137 @@ func run(opts *options, files []string) error {
 	return nil
 }
 
+// noPrint suppresses the output of the validation result.
+func noPrint(*csaf.RemoteValidationResult) {}
+
+// messageInstancePaths aggregates errors, warnings and infos by their
+// message.
+type messageInstancePaths struct {
+	message string
+	paths   []string
+}
+
+// messageInstancePathsList is a list for errors, warnings or infos.
+type messageInstancePathsList []messageInstancePaths
+
+// addAll adds all errors, warnings or infos of a test.
+func (mipl *messageInstancePathsList) addAll(rtrs []csaf.RemoteTestResult) {
+	for _, rtr := range rtrs {
+		mipl.add(rtr)
+	}
+}
+
+// add adds a test result unless it is a duplicate.
+func (mipl *messageInstancePathsList) add(rtr csaf.RemoteTestResult) {
+	for i := range *mipl {
+		m := &(*mipl)[i]
+		// Already have this message?
+		if m.message == rtr.Message {
+			for _, path := range m.paths {
+				// Avoid dupes.
+				if path == rtr.InstancePath {
+					return
+				}
+			}
+			m.paths = append(m.paths, rtr.InstancePath)
+			return
+		}
+	}
+	*mipl = append(*mipl, messageInstancePaths{
+		message: rtr.Message,
+		paths:   []string{rtr.InstancePath},
+	})
+}
+
+// print prints the details of the list to stdout if there are any.
+func (mipl messageInstancePathsList) print(info string) {
+	if len(mipl) == 0 {
+		return
+	}
+	fmt.Println(info)
+	for i := range mipl {
+		mip := &mipl[i]
+		fmt.Printf("  message: %s\n", mip.message)
+		fmt.Println("  instance path(s):")
+		for _, path := range mip.paths {
+			fmt.Printf("    %s\n", path)
+		}
+	}
+}
+
+// printShort outputs the validation result in an aggregated version.
+func printShort(rvr *csaf.RemoteValidationResult) {
+
+	var errors, warnings, infos messageInstancePathsList
+
+	for i := range rvr.Tests {
+		test := &rvr.Tests[i]
+		errors.addAll(test.Error)
+		warnings.addAll(test.Warning)
+		infos.addAll(test.Info)
+	}
+
+	fmt.Printf("isValid: %t\n", rvr.Valid)
+	errors.print("errors:")
+	warnings.print("warnings:")
+	infos.print("infos:")
+}
+
+// printImportant displays only the test results which are really relevant.
+func printImportant(rvr *csaf.RemoteValidationResult) {
+	printRemoteValidationResult(rvr, func(rt *csaf.RemoteTest) bool {
+		return !rt.Valid ||
+			len(rt.Info) > 0 || len(rt.Error) > 0 || len(rt.Warning) > 0
+	})
+}
+
+// printAll displays all test results.
+func printAll(rvr *csaf.RemoteValidationResult) {
+	printRemoteValidationResult(rvr, func(*csaf.RemoteTest) bool {
+		return true
+	})
+}
+
+// printInstanceAndMessages prints the message and the instance path of
+// a test result.
+func printInstanceAndMessages(info string, me []csaf.RemoteTestResult) {
+	if len(me) == 0 {
+		return
+	}
+	fmt.Printf("  %s\n", info)
+	for _, test := range me {
+		fmt.Printf("    instance path: %s\n", test.InstancePath)
+		fmt.Printf("    message: %s\n", test.Message)
+	}
+}
+
+// printRemoteValidationResult prints a filtered output of the remote validation result.
+func printRemoteValidationResult(
+	rvr *csaf.RemoteValidationResult,
+	accept func(*csaf.RemoteTest) bool,
+) {
+
+	fmt.Printf("isValid: %t\n", rvr.Valid)
+	fmt.Println("tests:")
+	nl := false
+	for i := range rvr.Tests {
+		test := &rvr.Tests[i]
+		if !accept(test) {
+			continue
+		}
+		if nl {
+			fmt.Println()
+		} else {
+			nl = true
+		}
+		fmt.Printf("  name: %s\n", test.Name)
+		fmt.Printf("  isValid: %t\n", test.Valid)
+		printInstanceAndMessages("errors:", test.Error)
+		printInstanceAndMessages("warnings:", test.Warning)
+		printInstanceAndMessages("infos:", test.Info)
+	}
+}
+
 func errCheck(err error) {
 	if err != nil {
 		if flags.WroteHelp(err) {

csaf/remotevalidation.go

@@ -10,10 +10,11 @@ package csaf
 
 import (
 	"bytes"
+	"compress/zlib"
 	"crypto/sha256"
 	"encoding/json"
-	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"sync"
 
@@ -32,12 +33,12 @@ var defaultPresets = []string{"mandatory"}
 
 var (
 	validationsBucket = []byte("validations")
-	validFalse        = []byte{0}
-	validTrue         = []byte{1}
+	cacheVersionKey   = []byte("version")
+	cacheVersion      = []byte("1")
 )
 
 // RemoteValidatorOptions are the configuation options
-// the remote validation service.
+// of the remote validation service.
 type RemoteValidatorOptions struct {
 	URL     string   `json:"url" toml:"url"`
 	Presets []string `json:"presets" toml:"presets"`
@@ -55,22 +56,37 @@ type outDocument struct {
 	Document any    `json:"document"`
 }
 
-// inDocument is the document recieved from the remote validation service.
-type inDocument struct {
-	Valid bool `json:"isValid"`
+// RemoteTestResult are any given test-result by a remote validator test.
+type RemoteTestResult struct {
+	Message      string `json:"message"`
+	InstancePath string `json:"instancePath"`
 }
 
-var errNotFound = errors.New("not found")
+// RemoteTest is the result of the remote tests
+// recieved by the remote validation service.
+type RemoteTest struct {
+	Name    string             `json:"name"`
+	Valid   bool               `json:"isValid"`
+	Error   []RemoteTestResult `json:"errors"`
+	Warning []RemoteTestResult `json:"warnings"`
+	Info    []RemoteTestResult `json:"infos"`
+}
+
+// RemoteValidationResult is the document recieved from the remote validation service.
+type RemoteValidationResult struct {
+	Valid bool         `json:"isValid"`
+	Tests []RemoteTest `json:"tests"`
+}
 
 type cache interface {
-	get(key []byte) (bool, error)
-	set(key []byte, valid bool) error
+	get(key []byte) ([]byte, error)
+	set(key []byte, value []byte) error
 	Close() error
 }
 
 // RemoteValidator validates an advisory document remotely.
 type RemoteValidator interface {
-	Validate(doc any) (bool, error)
+	Validate(doc any) (*RemoteValidationResult, error)
 	Close() error
 }
 
@@ -94,7 +110,7 @@ type syncedRemoteValidator struct {
 }
 
 // Validate implements the validation part of the RemoteValidator interface.
-func (srv *syncedRemoteValidator) Validate(doc any) (bool, error) {
+func (srv *syncedRemoteValidator) Validate(doc any) (*RemoteValidationResult, error) {
 	srv.Lock()
 	defer srv.Unlock()
 	return srv.RemoteValidator.Validate(doc)
@@ -122,7 +138,7 @@ func prepareTests(presets []string) []test {
 // prepareURL prepares the URL to be called for validation.
 func prepareURL(url string) string {
 	if url == "" {
-		return defaultURL + validationPath
+		url = defaultURL
 	}
 	return url + validationPath
 }
@@ -140,8 +156,34 @@ func prepareCache(config string) (cache, error) {
 
 	// Create the bucket.
 	if err := db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists(validationsBucket)
-		return err
+
+		// Create a new bucket with version set.
+		create := func() error {
+			b, err := tx.CreateBucket(validationsBucket)
+			if err != nil {
+				return err
+			}
+			if err := b.Put(cacheVersionKey, cacheVersion); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		b := tx.Bucket(validationsBucket)
+
+		if b == nil { // Bucket does not exists -> create.
+			return create()
+		}
+		// Bucket exists.
+		if v := b.Get(cacheVersionKey); !bytes.Equal(v, cacheVersion) {
+			// version mismatch -> delete and re-create.
+			if err := tx.DeleteBucket(validationsBucket); err != nil {
+				return err
+			}
+			return create()
+		}
+		return nil
+
 	}); err != nil {
 		db.Close()
 		return nil, err
@@ -154,31 +196,23 @@ func prepareCache(config string) (cache, error) {
 type boltCache struct{ *bolt.DB }
 
 // get implements the fetch part of the cache interface.
-func (bc boltCache) get(key []byte) (valid bool, err error) {
-	err2 := bc.View(func(tx *bolt.Tx) error {
+func (bc boltCache) get(key []byte) ([]byte, error) {
+	var value []byte
+	if err := bc.View(func(tx *bolt.Tx) error {
 		b := tx.Bucket(validationsBucket)
-		v := b.Get(key)
-		if v == nil {
-			err = errNotFound
-		} else {
-			valid = v[0] != 0
-		}
+		value = b.Get(key)
 		return nil
-	})
-	if err2 != nil {
-		err = err2
+	}); err != nil {
+		return nil, err
 	}
-	return
+	return value, nil
 }
 
-// get implements the store part of the cache interface.
-func (bc boltCache) set(key []byte, valid bool) error {
+// set implements the store part of the cache interface.
+func (bc boltCache) set(key, value []byte) error {
 	return bc.Update(func(tx *bolt.Tx) error {
 		b := tx.Bucket(validationsBucket)
-		if valid {
-			return b.Put(key, validTrue)
-		}
-		return b.Put(key, validFalse)
+		return b.Put(key, value)
 	})
 }
 
@@ -217,22 +251,37 @@ func (v *remoteValidator) key(doc any) ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
+// deserialize revives a remote validation result from a cache value.
+func deserialize(value []byte) (*RemoteValidationResult, error) {
+	r, err := zlib.NewReader(bytes.NewReader(value))
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+	var rvr RemoteValidationResult
+	if err := json.NewDecoder(r).Decode(&rvr); err != nil {
+		return nil, err
+	}
+	return &rvr, nil
+}
+
 // Validate executes a remote validation of an advisory.
-func (v *remoteValidator) Validate(doc any) (bool, error) {
+func (v *remoteValidator) Validate(doc any) (*RemoteValidationResult, error) {
 
 	var key []byte
 
+	// First look into cache.
 	if v.cache != nil {
 		var err error
 		if key, err = v.key(doc); err != nil {
-			return false, err
+			return nil, err
 		}
-		valid, err := v.cache.get(key)
-		if err != errNotFound {
-			if err != nil {
-				return false, err
-			}
-			return valid, nil
+		value, err := v.cache.get(key)
+		if err != nil {
+			return nil, err
+		}
+		if value != nil {
+			return deserialize(value)
 		}
 	}
 
@@ -243,7 +292,7 @@ func (v *remoteValidator) Validate(doc any) (bool, error) {
 
 	var buf bytes.Buffer
 	if err := json.NewEncoder(&buf).Encode(&o); err != nil {
-		return false, err
+		return nil, err
 	}
 
 	resp, err := http.Post(
@@ -252,30 +301,46 @@ func (v *remoteValidator) Validate(doc any) (bool, error) {
 		bytes.NewReader(buf.Bytes()))
 
 	if err != nil {
-		return false, err
+		return nil, err
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return false, fmt.Errorf(
+		return nil, fmt.Errorf(
 			"POST failed: %s (%d)", resp.Status, resp.StatusCode)
 	}
 
-	valid, err := func() (bool, error) {
-		defer resp.Body.Close()
-		var in inDocument
-		return in.Valid, json.NewDecoder(resp.Body).Decode(&in)
-	}()
+	var (
+		zout *zlib.Writer
+		rvr  RemoteValidationResult
+	)
 
-	if err != nil {
-		return false, err
+	if err := func() error {
+		defer resp.Body.Close()
+		var in io.Reader
+		// If we are caching record the incoming data and compress it.
+		if key != nil {
+			buf.Reset() // reuse the out buffer.
+			zout = zlib.NewWriter(&buf)
+			in = io.TeeReader(resp.Body, zout)
+		} else {
+			// no cache -> process directly.
+			in = resp.Body
+		}
+		return json.NewDecoder(in).Decode(&rvr)
+	}(); err != nil {
+		return nil, err
 	}
 
+	// Store in cache
 	if key != nil {
-		// store in cache
-		if err := v.cache.set(key, valid); err != nil {
-			return valid, err
+		if err := zout.Close(); err != nil {
+			return nil, err
+		}
+		// The document is now compressed in the buffer.
+		if err := v.cache.set(key, buf.Bytes()); err != nil {
+			return nil, err
 		}
 	}
 
-	return valid, nil
+	return &rvr, nil
 }

docs/csaf_validator.md

@@ -8,11 +8,17 @@ is a tool to validate local advisories files against the JSON Schema and an opti
 csaf_validator [OPTIONS] files...
 
 Application Options:
-      --version                Display version of the binary
-      --validator=URL          URL to validate documents remotely
-      --validatorcache=FILE    FILE to cache remote validations
-      --validatorpreset=       One or more presets to validate remotely (default: mandatory)
+      --version                   Display version of the binary
+      --validator=URL             URL to validate documents remotely
+      --validatorcache=FILE       FILE to cache remote validations
+      --validatorpreset=          One or more presets to validate remotely (default: mandatory)
+      -o AMOUNT, --output=AMOUNT  If a remote validator was used, display the results in JSON format
+
+AMOUNT:
+ all: Print the entire JSON output
+ important: Print the entire JSON output but omit all tests without errors, warnings and infos.
+ short: Print only the result, errors, warnings and infos.
 
 Help Options:
-  -h, --help                   Show this help message
+  -h, --help                      Show this help message
 ```