Merge branch 'main' of github.com:csaf-poc/csaf_distribution into main

cmd/csaf_aggregator/processor.go

@@ -9,14 +9,10 @@
 package main
 
 import (
-	"encoding/json"
-	"errors"
-	"io"
+	"fmt"
 	"log"
-	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 	"github.com/csaf-poc/csaf_distribution/csaf"
@@ -75,78 +71,24 @@ func (w *worker) createDir() (string, error) {
 	return dir, err
 }
 
-// httpsDomain prefixes a domain with 'https://'.
-func httpsDomain(domain string) string {
-	if strings.HasPrefix(domain, "https://") {
-		return domain
-	}
-	return "https://" + domain
-}
-
-var providerMetadataLocations = [...]string{
-	".well-known/csaf",
-	"security/data/csaf",
-	"advisories/csaf",
-	"security/csaf",
-}
-
 func (w *worker) locateProviderMetadata(domain string) error {
 
-	w.metadataProvider = nil
+	lpmd := csaf.LoadProviderMetadataForDomain(
+		w.client, domain, func(format string, args ...interface{}) {
+			log.Printf(
+				"Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...)
+		})
 
-	download := func(r io.Reader) error {
-		if err := json.NewDecoder(r).Decode(&w.metadataProvider); err != nil {
-			log.Printf("error: %s\n", err)
-			return errNotFound
+	if lpmd == nil {
+		return fmt.Errorf("no provider-metadata.json found for '%s'", domain)
 	}
+
+	w.metadataProvider = lpmd.Document
+	w.loc = lpmd.URL
+
 	return nil
 }
 
-	hd := httpsDomain(domain)
-	for _, loc := range providerMetadataLocations {
-		url := hd + "/" + loc
-		if err := downloadJSON(w.client, url, download); err != nil {
-			if err == errNotFound {
-				continue
-			}
-			return err
-		}
-		if w.metadataProvider != nil {
-			w.loc = loc
-			return nil
-		}
-	}
-
-	// Read from security.txt
-
-	path := hd + "/.well-known/security.txt"
-	res, err := w.client.Get(path)
-	if err != nil {
-		return err
-	}
-
-	if res.StatusCode != http.StatusOK {
-		return errNotFound
-	}
-
-	if err := func() error {
-		defer res.Body.Close()
-		urls, err := csaf.ExtractProviderURL(res.Body, false)
-		if err != nil {
-			return err
-		}
-		if len(urls) == 0 {
-			return errors.New("no provider-metadata.json found in secturity.txt")
-		}
-		w.loc = urls[0]
-		return nil
-	}(); err != nil {
-		return err
-	}
-
-	return downloadJSON(w.client, w.loc, download)
-}
-
 // removeOrphans removes the directories that are not in the providers list.
 func (p *processor) removeOrphans() error {
 

cmd/csaf_checker/main.go

@@ -146,7 +146,7 @@ func main() {
 		return
 	}
 
-	if (opts.ClientCert != nil && opts.ClientKey == nil) || (opts.ClientCert == nil && opts.ClientKey != nil) {
+	if opts.ClientCert != nil && opts.ClientKey == nil || opts.ClientCert == nil && opts.ClientKey != nil {
 		log.Println("Both client-key and client-cert options must be set for the authentication.")
 		return
 	}

cmd/csaf_checker/processor.go

@@ -795,111 +795,6 @@ func (p *processor) checkListing(string) error {
 	return nil
 }
 
-var providerMetadataLocations = [...]string{
-	".well-known/csaf",
-	"security/data/csaf",
-	"advisories/csaf",
-	"security/csaf",
-}
-
-// locateProviderMetadata searches for provider-metadata.json at various
-// locations mentioned in "7.1.7 Requirement 7: provider-metadata.json".
-func (p *processor) locateProviderMetadata(
-	domain string,
-	found func(string, io.Reader) error,
-) error {
-
-	client := p.httpClient()
-	tryURL := func(url string) (bool, error) {
-		log.Printf("Trying: %v\n", url)
-		res, err := client.Get(url)
-
-		if err != nil || res.StatusCode != http.StatusOK ||
-			res.Header.Get("Content-Type") != "application/json" {
-			// ignore this as it is expected.
-			return false, nil
-		}
-
-		if err := func() error {
-			defer res.Body.Close()
-			return found(url, res.Body)
-		}(); err != nil {
-			return false, err
-		}
-		return true, nil
-	}
-
-	for _, loc := range providerMetadataLocations {
-		url := "https://" + domain + "/" + loc + "/provider-metadata.json"
-		ok, err := tryURL(url)
-		if err != nil {
-			if err == errContinue {
-				continue
-			}
-			return err
-		}
-		if ok {
-			return nil
-		}
-	}
-
-	// Read from security.txt
-
-	path := "https://" + domain + "/.well-known/security.txt"
-	log.Printf("Searching in: %v\n", path)
-	res, err := client.Get(path)
-	if err == nil && res.StatusCode == http.StatusOK {
-		loc, err := func() (string, error) {
-			defer res.Body.Close()
-			return p.extractProviderURL(res.Body)
-		}()
-
-		if err != nil {
-			log.Printf("did not find provider URL in /.well-known/security.txt, error: %v\n", err)
-		}
-
-		if loc != "" {
-			if _, err = tryURL(loc); err == errContinue {
-				err = nil
-			}
-			return err
-		}
-	}
-
-	// Read from DNS path
-
-	path = "https://csaf.data.security." + domain
-	ok, err := tryURL(path)
-	if err != nil {
-		return err
-	}
-	if ok {
-		return nil
-	}
-
-	return errStop
-}
-
-func (p *processor) extractProviderURL(r io.Reader) (string, error) {
-	urls, err := csaf.ExtractProviderURL(r, true)
-	if err != nil {
-		return "", err
-	}
-	if len(urls) == 0 {
-		return "", errors.New("no provider-metadata.json found")
-	}
-
-	if len(urls) > 1 {
-		p.badSecurity.use()
-		p.badSecurity.add("Found %d CSAF entries in security.txt", len(urls))
-	}
-	if !strings.HasPrefix(urls[0], "https://") {
-		p.badSecurity.use()
-		p.badSecurity.add("CSAF URL does not start with https://: %s", urls[0])
-	}
-	return urls[0], nil
-}
-
 // checkProviderMetadata checks provider-metadata.json. If it exists,
 // decodes, and validates against the JSON schema.
 // According to the result, the respective error messages added to
@@ -909,44 +804,20 @@ func (p *processor) checkProviderMetadata(domain string) error {
 
 	p.badProviderMetadata.use()
 
-	found := func(url string, content io.Reader) error {
+	client := p.httpClient()
 
-		// Calculate checksum for later comparison.
-		hash := sha256.New()
+	lpmd := csaf.LoadProviderMetadataForDomain(client, domain, p.badProviderMetadata.add)
 
-		tee := io.TeeReader(content, hash)
-		if err := json.NewDecoder(tee).Decode(&p.pmd); err != nil {
-			p.badProviderMetadata.add("%s: Decoding JSON failed: %v", url, err)
-			return errContinue
-		}
-
-		p.pmd256 = hash.Sum(nil)
-
-		errors, err := csaf.ValidateProviderMetadata(p.pmd)
-		if err != nil {
-			return err
-		}
-		if len(errors) > 0 {
-			p.badProviderMetadata.add("%s: Validating against JSON schema failed:", url)
-			for _, msg := range errors {
-				p.badProviderMetadata.add(strings.ReplaceAll(msg, `%`, `%%`))
-			}
+	if lpmd == nil {
+		p.badProviderMetadata.add("No valid provider-metadata.json found.")
 		p.badProviderMetadata.add("STOPPING here - cannot perform other checks.")
 		return errStop
 	}
-		p.pmdURL = url
-		return nil
-	}
 
-	if err := p.locateProviderMetadata(domain, found); err != nil {
-		return err
-	}
+	p.pmdURL = lpmd.URL
+	p.pmd256 = lpmd.Hash
+	p.pmd = lpmd.Document
 
-	if p.pmdURL == "" {
-		p.badProviderMetadata.add("No provider-metadata.json found.")
-		p.badProviderMetadata.add("STOPPING here - cannot perform other checks.")
-		return errStop
-	}
 	return nil
 }
 

cmd/csaf_uploader/main.go

@@ -392,7 +392,7 @@ func main() {
 		check(readInteractive("Enter OpenPGP passphrase: ", &opts.Passphrase))
 	}
 
-	if (opts.ClientCert != nil && opts.ClientKey == nil) || (opts.ClientCert == nil && opts.ClientKey != nil) {
+	if opts.ClientCert != nil && opts.ClientKey == nil || opts.ClientCert == nil && opts.ClientKey != nil {
 		log.Println("Both client-key and client-cert options must be set for the authentication.")
 		return
 	}

csaf/util.go

@@ -10,10 +10,226 @@ package csaf
 
 import (
 	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
 	"io"
+	"log"
+	"net/http"
 	"strings"
+
+	"github.com/csaf-poc/csaf_distribution/util"
 )
 
+// LoadedProviderMetadata represents a loaded provider metadata.
+type LoadedProviderMetadata struct {
+	// URL is location where the document was found.
+	URL string
+	// Document is the de-serialized JSON document.
+	Document interface{}
+	// Hash is a SHA256 sum over the document.
+	Hash []byte
+	// Messages are the error message happened while loading.
+	Messages []string
+}
+
+// LoadProviderMetadataFromURL loads a provider metadata from a given URL.
+// Returns nil if the document was not found.
+func LoadProviderMetadataFromURL(client util.Client, url string) *LoadedProviderMetadata {
+
+	res, err := client.Get(url)
+
+	if err != nil || res.StatusCode != http.StatusOK {
+		// Treat as not found.
+		return nil
+	}
+
+	// TODO: Check for application/json and log it.
+
+	defer res.Body.Close()
+
+	// Calculate checksum for later comparison.
+	hash := sha256.New()
+
+	result := LoadedProviderMetadata{URL: url}
+
+	tee := io.TeeReader(res.Body, hash)
+
+	if err := json.NewDecoder(tee).Decode(&result.Document); err != nil {
+		result.Messages = []string{fmt.Sprintf("%s: Decoding JSON failed: %v", url, err)}
+		return &result
+	}
+
+	result.Hash = hash.Sum(nil)
+
+	errors, err := ValidateProviderMetadata(result.Document)
+	if err != nil {
+		result.Messages = []string{
+			fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)}
+		return &result
+	}
+
+	if len(errors) > 0 {
+		result.Messages = []string{
+			fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)}
+		for _, msg := range errors {
+			result.Messages = append(result.Messages, strings.ReplaceAll(msg, `%`, `%%`))
+		}
+	}
+
+	return &result
+}
+
+// LoadProviderMetadatasFromSecurity loads a secturity.txt,
+// extracts and the CSAF urls from the document.
+// Returns nil if no url was successfully found.
+func LoadProviderMetadatasFromSecurity(client util.Client, path string) []*LoadedProviderMetadata {
+
+	res, err := client.Get(path)
+
+	if err != nil || res.StatusCode != http.StatusOK {
+		// Treat as not found.
+		return nil
+	}
+
+	// Extract all potential URLs from CSAF.
+	urls, err := func() ([]string, error) {
+		defer res.Body.Close()
+		return ExtractProviderURL(res.Body, true)
+	}()
+
+	if err != nil {
+		// Treat as not found
+		return nil
+	}
+
+	var results []*LoadedProviderMetadata
+
+	// Load the URLs
+	for _, url := range urls {
+		if result := LoadProviderMetadataFromURL(client, url); result != nil {
+			results = append(results, result)
+		}
+	}
+
+	return results
+}
+
+// LoadProviderMetadataForDomain loads a provider metadata for a given domain.
+// Returns nil if no provider metadata was found.
+// The logging can be use to track the errors happening while loading.
+func LoadProviderMetadataForDomain(
+	client util.Client,
+	domain string,
+	logging func(format string, args ...interface{}),
+) *LoadedProviderMetadata {
+
+	if logging == nil {
+		logging = func(format string, args ...interface{}) {
+			log.Printf("FindProviderMetadata: "+format+"\n", args...)
+		}
+	}
+
+	// Valid provider metadata under well-known.
+	var wellknownGood *LoadedProviderMetadata
+
+	// First try well-know path
+	wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
+	log.Printf("Trying: %s\n", wellknownURL)
+	wellknownResult := LoadProviderMetadataFromURL(client, wellknownURL)
+
+	if wellknownResult == nil {
+		logging("%s not found.", wellknownURL)
+	} else if len(wellknownResult.Messages) > 0 {
+		// There are issues
+		for _, msg := range wellknownResult.Messages {
+			logging(msg)
+		}
+	} else {
+		// We have a candidate.
+		wellknownGood = wellknownResult
+	}
+
+	// Next load the PMDs from security.txt
+	secURL := "https://" + domain + "/.well-known/security.txt"
+	log.Printf("Trying: %s\n", secURL)
+	secResults := LoadProviderMetadatasFromSecurity(client, secURL)
+
+	if secResults == nil {
+		logging("%s failed to load.", secURL)
+	} else {
+		// Filter out the results which are valid.
+		var secGoods []*LoadedProviderMetadata
+
+		for _, result := range secResults {
+			if len(result.Messages) > 0 {
+				for _, msg := range result.Messages {
+					logging(msg)
+				}
+			} else {
+				secGoods = append(secGoods, result)
+			}
+		}
+
+		// security.txt contains good entries.
+		if len(secGoods) > 0 {
+			// we have a wellknown good take it.
+			if wellknownGood != nil {
+				// check if first of security urls is identical to wellknown.
+				if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) {
+					// Mention extra CSAF entries
+					for _, extra := range secGoods[1:] {
+						logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL)
+					}
+				} else {
+					// Complaint about not matching.
+					logging("First entry of security.txt and well-known don't match.")
+					// List all the security urls.
+					for _, sec := range secGoods {
+						logging("Ignoring CSAF entry in security.txt: %s", sec.URL)
+					}
+				}
+				// Take the good well-known.
+				return wellknownGood
+			}
+
+			// Don't have well-known. Take first good from security.txt.
+			// Mention extra CSAF entries
+			for _, extra := range secGoods[1:] {
+				logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL)
+			}
+
+			return secGoods[0]
+		}
+	}
+
+	// If we have a good well-known take it.
+	if wellknownGood != nil {
+		return wellknownGood
+	}
+
+	// Last resort fall back to DNS.
+
+	dnsURL := "https://csaf.data.security." + domain
+	log.Printf("Trying: %s\n", dnsURL)
+	dnsResult := LoadProviderMetadataFromURL(client, dnsURL)
+
+	if dnsResult == nil {
+		logging("%s not found.", dnsURL)
+	} else if len(dnsResult.Messages) > 0 {
+		for _, msg := range dnsResult.Messages {
+			logging(msg)
+		}
+	} else {
+		// DNS seems to be okay.
+		return dnsResult
+	}
+
+	// We failed all.
+	return nil
+}
+
 // ExtractProviderURL extracts URLs of provider metadata.
 // If all is true all URLs are returned. Otherwise only the first is returned.
 func ExtractProviderURL(r io.Reader, all bool) ([]string, error) {