Merge pull request #366 from csaf-poc/cleanup_provider_metadata_loading

Prepare infrastructure for role based reporting

cmd/csaf_aggregator/processor.go

@@ -79,11 +79,17 @@ func (w *worker) createDir() (string, error) {
 
 func (w *worker) locateProviderMetadata(domain string) error {
 
-	lpmd := csaf.LoadProviderMetadataForDomain(
+	loader := csaf.NewProviderMetadataLoader(w.client)
-		w.client, domain, func(format string, args ...any) {
+
+	lpmd := loader.Load(domain)
+
+	if w.processor.cfg.Verbose {
+		for i := range lpmd.Messages {
 			log.Printf(
-				"Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...)
+				"Loading provider-metadata.json of %q: %s\n",
-		})
+				domain, lpmd.Messages[i].Message)
+		}
+	}
 
 	if !lpmd.Valid() {
 		return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain)

cmd/csaf_checker/main.go

@@ -140,28 +140,6 @@ func writeReport(report *Report, opts *options) error {
 	return writer(report, w)
 }
 
-// buildReporters initializes each report by assigning a number and description to it.
-// It returns an array of the reporter interface type.
-func buildReporters() []reporter {
-	return []reporter{
-		&validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}},
-		&filenameReporter{baseReporter{num: 2, description: "Filename"}},
-		&tlsReporter{baseReporter{num: 3, description: "TLS"}},
-		&redirectsReporter{baseReporter{num: 6, description: "Redirects"}},
-		&providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}},
-		&securityReporter{baseReporter{num: 8, description: "security.txt"}},
-		&wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}},
-		&dnsPathReporter{baseReporter{num: 10, description: "DNS path"}},
-		&oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}},
-		&indexReporter{baseReporter{num: 12, description: "index.txt"}},
-		&changesReporter{baseReporter{num: 13, description: "changes.csv"}},
-		&directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}},
-		&integrityReporter{baseReporter{num: 18, description: "Integrity"}},
-		&signaturesReporter{baseReporter{num: 19, description: "Signatures"}},
-		&publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}},
-	}
-}
-
 // run uses a processor to check all the given domains or direct urls
 // and generates a report.
 func run(opts *options, domains []string) (*Report, error) {
@@ -170,7 +148,7 @@ func run(opts *options, domains []string) (*Report, error) {
 		return nil, err
 	}
 	defer p.close()
-	return p.run(buildReporters(), domains)
+	return p.run(domains)
 }
 
 func main() {

cmd/csaf_checker/processor.go

@@ -223,7 +223,7 @@ func (p *processor) clean() {
 // run calls checkDomain function for each domain in the given "domains" parameter.
 // Then it calls the report method on each report from the given "reporters" parameter for each domain.
 // It returns a pointer to the report and nil, otherwise an error.
-func (p *processor) run(reporters []reporter, domains []string) (*Report, error) {
+func (p *processor) run(domains []string) (*Report, error) {
 
 	report := Report{
 		Date:    ReportTime{Time: time.Now().UTC()},
@@ -231,19 +231,24 @@ func (p *processor) run(reporters []reporter, domains []string) (*Report, error)
 	}
 
 	for _, d := range domains {
+		if p.checkProviderMetadata(d) {
 			if err := p.checkDomain(d); err != nil {
 				if err == errContinue || err == errStop {
 					continue
 				}
 				return nil, err
 			}
-		domain := &Domain{Name: d}
-		for _, r := range reporters {
-			r.report(p, domain)
 		}
+		domain := &Domain{Name: d}
 
 		if err := p.fillMeta(domain); err != nil {
 			log.Printf("Filling meta data failed: %v\n", err)
+			// reporters depend on role.
+			continue
+		}
+
+		for _, r := range buildReporters(*domain.Role) {
+			r.report(p, domain)
 		}
 
 		report.Domains = append(report.Domains, domain)
@@ -287,7 +292,6 @@ func (p *processor) domainChecks(domain string) []func(*processor, string) error
 	direct := strings.HasPrefix(domain, "https://")
 
 	checks := []func(*processor, string) error{
-		(*processor).checkProviderMetadata,
 		(*processor).checkPGPKeys,
 	}
 
@@ -1113,26 +1117,32 @@ func (p *processor) checkListing(string) error {
 // decodes, and validates against the JSON schema.
 // According to the result, the respective error messages added to
 // badProviderMetadata.
-// It returns nil if all checks are passed.
+func (p *processor) checkProviderMetadata(domain string) bool {
-func (p *processor) checkProviderMetadata(domain string) error {
 
 	p.badProviderMetadata.use()
 
 	client := p.httpClient()
 
-	lpmd := csaf.LoadProviderMetadataForDomain(client, domain, p.badProviderMetadata.warn)
+	loader := csaf.NewProviderMetadataLoader(client)
+
+	lpmd := loader.Load(domain)
+
+	for i := range lpmd.Messages {
+		// TODO: Filter depending on the role.
+		p.badProviderMetadata.error(lpmd.Messages[i].Message)
+	}
 
 	if !lpmd.Valid() {
 		p.badProviderMetadata.error("No valid provider-metadata.json found.")
 		p.badProviderMetadata.error("STOPPING here - cannot perform other checks.")
-		return errStop
+		return false
 	}
 
 	p.pmdURL = lpmd.URL
 	p.pmd256 = lpmd.Hash
 	p.pmd = lpmd.Document
 
-	return nil
+	return true
 }
 
 // checkSecurity checks the security.txt file by making HTTP request to fetch it.

cmd/csaf_checker/reporters.go

@@ -12,6 +12,8 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+
+	"github.com/csaf-poc/csaf_distribution/csaf"
 )
 
 type (
@@ -22,6 +24,8 @@ type (
 	validReporter             struct{ baseReporter }
 	filenameReporter          struct{ baseReporter }
 	tlsReporter               struct{ baseReporter }
+	tlpWhiteReporter          struct{ baseReporter }
+	tlpAmberRedReporter       struct{ baseReporter }
 	redirectsReporter         struct{ baseReporter }
 	providerMetadataReport    struct{ baseReporter }
 	securityReporter          struct{ baseReporter }
@@ -31,11 +35,83 @@ type (
 	indexReporter             struct{ baseReporter }
 	changesReporter           struct{ baseReporter }
 	directoryListingsReporter struct{ baseReporter }
+	rolieFeedReporter         struct{ baseReporter }
+	rolieServiceReporter      struct{ baseReporter }
+	rolieCategoryReporter     struct{ baseReporter }
 	integrityReporter         struct{ baseReporter }
 	signaturesReporter        struct{ baseReporter }
 	publicPGPKeyReporter      struct{ baseReporter }
+	listReporter              struct{ baseReporter }
+	hasTwoReporter            struct{ baseReporter }
+	mirrorReporter            struct{ baseReporter }
 )
 
+var reporters = [23]reporter{
+	&validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}},
+	&filenameReporter{baseReporter{num: 2, description: "Filename"}},
+	&tlsReporter{baseReporter{num: 3, description: "TLS"}},
+	&tlpWhiteReporter{baseReporter{num: 4, description: "TLP:WHITE"}},
+	&tlpAmberRedReporter{baseReporter{num: 5, description: "TLP:AMBER and TLP:RED"}},
+	&redirectsReporter{baseReporter{num: 6, description: "Redirects"}},
+	&providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}},
+	&securityReporter{baseReporter{num: 8, description: "security.txt"}},
+	&wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}},
+	&dnsPathReporter{baseReporter{num: 10, description: "DNS path"}},
+	&oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}},
+	&indexReporter{baseReporter{num: 12, description: "index.txt"}},
+	&changesReporter{baseReporter{num: 13, description: "changes.csv"}},
+	&directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}},
+	&rolieFeedReporter{baseReporter{num: 15, description: "ROLIE feed"}},
+	&rolieServiceReporter{baseReporter{num: 16, description: "ROLIE service document"}},
+	&rolieCategoryReporter{baseReporter{num: 17, description: "ROLIE category document"}},
+	&integrityReporter{baseReporter{num: 18, description: "Integrity"}},
+	&signaturesReporter{baseReporter{num: 19, description: "Signatures"}},
+	&publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}},
+	&listReporter{baseReporter{num: 21, description: "List of CSAF providers"}},
+	&hasTwoReporter{baseReporter{num: 22, description: "Two disjoint issuing parties"}},
+	&mirrorReporter{baseReporter{num: 23, description: "Mirror"}},
+}
+
+var roleImplies = map[csaf.MetadataRole][]csaf.MetadataRole{
+	csaf.MetadataRoleProvider:        {csaf.MetadataRolePublisher},
+	csaf.MetadataRoleTrustedProvider: {csaf.MetadataRoleProvider},
+}
+
+func requirements(role csaf.MetadataRole) [][2]int {
+	var own [][2]int
+	switch role {
+	case csaf.MetadataRoleTrustedProvider:
+		own = [][2]int{{18, 20}}
+	case csaf.MetadataRoleProvider:
+		// TODO: use commented numbers when TLPs should be checked.
+		own = [][2]int{{6 /* 5 */, 7}, {8, 10}, {11, 14}, {15, 17}}
+	case csaf.MetadataRolePublisher:
+		own = [][2]int{{1, 3 /* 4 */}}
+	}
+	for _, base := range roleImplies[role] {
+		own = append(own, requirements(base)...)
+	}
+	return own
+}
+
+// buildReporters initializes each report by assigning a number and description to it.
+// It returns an array of the reporter interface type.
+func buildReporters(role csaf.MetadataRole) []reporter {
+	var reps []reporter
+	reqs := requirements(role)
+	// sort to have them ordered by there number.
+	sort.Slice(reqs, func(i, j int) bool { return reqs[i][0] < reqs[j][0] })
+	for _, req := range reqs {
+		from, to := req[0]-1, req[1]-1
+		for i := from; i <= to; i++ {
+			if rep := reporters[i]; rep != nil {
+				reps = append(reps, rep)
+			}
+		}
+	}
+	return reps
+}
+
 func (bc *baseReporter) requirement(domain *Domain) *Requirement {
 	req := &Requirement{
 		Num:         bc.num,
@@ -115,6 +191,21 @@ func (r *tlsReporter) report(p *processor, domain *Domain) {
 	req.message(ErrorType, urls...)
 }
 
+// report tests if a document labeled TLP:WHITE
+// is freely accessible and sets the "message" field value
+// of the "Requirement" struct as a result of that.
+func (r *tlpWhiteReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}
+
+// report tests if a document labeled TLP:AMBER
+// or TLP:RED is access protected
+// and sets the "message" field value
+// of the "Requirement" struct as a result of that.
+func (r *tlpAmberRedReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}
+
 // report tests if redirects are used and sets the "message" field value
 // of the "Requirement" struct as a result of that.
 func (r *redirectsReporter) report(p *processor, domain *Domain) {
@@ -269,6 +360,31 @@ func (r *directoryListingsReporter) report(p *processor, domain *Domain) {
 	req.Messages = p.badDirListings
 }
 
+// report checks whether there is only a single ROLIE feed for a
+// given TLP level and whether any of the TLP levels
+// TLP:WHITE, TLP:GREEN or unlabeled exists and sets the "message" field value
+// of the "Requirement" struct as a result of that.
+func (r *rolieFeedReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}
+
+// report tests whether a ROLIE service document is used and if so,
+// whether it is a [RFC8322] conform JSON file that lists the
+// ROLIE feed documents and sets the "message" field value
+// of the "Requirement" struct as a result of that.
+func (r *rolieServiceReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}
+
+// report tests whether a ROLIE category document is used and if so,
+// whether it is a [RFC8322] conform JSON file and is used to dissect
+// documents by certain criteria
+// and sets the "message" field value
+// of the "Requirement" struct as a result of that.
+func (r *rolieCategoryReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}
+
 func (r *integrityReporter) report(p *processor, domain *Domain) {
 	req := r.requirement(domain)
 	if !p.badIntegrities.used() {
@@ -306,3 +422,25 @@ func (r *publicPGPKeyReporter) report(p *processor, domain *Domain) {
 			p.keys.CountEntities()))
 	}
 }
+
+// report tests whether a CSAF aggregator JSON schema conform
+// aggregator.json exists without being adjacent to a
+// provider-metadata.json
+func (r *listReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}
+
+// report tests whether the aggregator.json lists at least
+// two disjoint issuing parties. TODO: reevaluate phrasing (Req 7.1.22)
+func (r *hasTwoReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}
+
+// report tests whether the CSAF documents of each issuing mirrored party
+// is in a different folder, which are adjacent to the aggregator.json and
+// if the folder name is retrieved from the name of the issuing authority.
+// It also tests whether each folder has a provider-metadata.json for their
+// party and provides ROLIE feed documents.
+func (r *mirrorReporter) report(_ *processor, _ *Domain) {
+	// TODO
+}

cmd/csaf_downloader/downloader.go

@@ -118,11 +118,16 @@ func (d *downloader) httpClient() util.Client {
 func (d *downloader) download(ctx context.Context, domain string) error {
 	client := d.httpClient()
 
-	lpmd := csaf.LoadProviderMetadataForDomain(
+	loader := csaf.NewProviderMetadataLoader(client)
-		client, domain, func(format string, args ...any) {
+
-			log.Printf(
+	lpmd := loader.Load(domain)
-				"Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...)
+
-		})
+	if d.opts.Verbose {
+		for i := range lpmd.Messages {
+			log.Printf("Loading provider-metadata.json for %q: %s\n",
+				domain, lpmd.Messages[i].Message)
+		}
+	}
 
 	if !lpmd.Valid() {
 		return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain)

csaf/providermetaloader.go

@@ -0,0 +1,328 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
+
+package csaf
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/csaf-poc/csaf_distribution/util"
+)
+
+// ProviderMetadataLoader helps load provider-metadata.json from
+// the various locations.
+type ProviderMetadataLoader struct {
+	client   util.Client
+	already  map[string]*LoadedProviderMetadata
+	messages ProviderMetadataLoadMessages
+}
+
+// ProviderMetadataLoadMessageType is the type of the message.
+type ProviderMetadataLoadMessageType int
+
+const (
+	//JSONDecodingFailed indicates problems with JSON decoding
+	JSONDecodingFailed ProviderMetadataLoadMessageType = iota
+	// SchemaValidationFailed indicates a general problem with schema validation.
+	SchemaValidationFailed
+	// SchemaValidationFailedDetail is a failure detail in schema validation.
+	SchemaValidationFailedDetail
+	// HTTPFailed indicates that loading on HTTP level failed.
+	HTTPFailed
+	// ExtraProviderMetadataFound indicates an extra PMD found in security.txt.
+	ExtraProviderMetadataFound
+	// WellknownSecurityMismatch indicates that the PMDs found under wellknown and
+	// in the security do not match.
+	WellknownSecurityMismatch
+	// IgnoreProviderMetadata indicates that a extra PMD was ignored.
+	IgnoreProviderMetadata
+)
+
+// ProviderMetadataLoadMessage is a message generated while loading
+// a provider meta data file.
+type ProviderMetadataLoadMessage struct {
+	Type    ProviderMetadataLoadMessageType
+	Message string
+}
+
+// ProviderMetadataLoadMessages is a list of loading messages.
+type ProviderMetadataLoadMessages []ProviderMetadataLoadMessage
+
+// LoadedProviderMetadata represents a loaded provider metadata.
+type LoadedProviderMetadata struct {
+	// URL is location where the document was found.
+	URL string
+	// Document is the de-serialized JSON document.
+	Document any
+	// Hash is a SHA256 sum over the document.
+	Hash []byte
+	// Messages are the error message happened while loading.
+	Messages ProviderMetadataLoadMessages
+}
+
+// Add appends a message to the list of loading messages.
+func (pmlm *ProviderMetadataLoadMessages) Add(
+	typ ProviderMetadataLoadMessageType,
+	msg string,
+) {
+	*pmlm = append(*pmlm, ProviderMetadataLoadMessage{
+		Type:    typ,
+		Message: msg,
+	})
+}
+
+// AppendUnique appends unique messages from a second list.
+func (pmlm *ProviderMetadataLoadMessages) AppendUnique(other ProviderMetadataLoadMessages) {
+next:
+	for _, o := range other {
+		for _, m := range *pmlm {
+			if m == o {
+				continue next
+			}
+		}
+		*pmlm = append(*pmlm, o)
+	}
+}
+
+// Valid returns true if the loaded document is valid.
+func (lpm *LoadedProviderMetadata) Valid() bool {
+	return lpm != nil && lpm.Document != nil && lpm.Hash != nil
+}
+
+// NewProviderMetadataLoader create a new loader.
+func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader {
+	return &ProviderMetadataLoader{
+		client:  client,
+		already: map[string]*LoadedProviderMetadata{},
+	}
+}
+
+// Load loads a provider metadata for a given path.
+// If the domain starts with `https://` it only attemps to load
+// the data from that URL.
+func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata {
+
+	// Check direct path
+	if strings.HasPrefix(domain, "https://") {
+		return pmdl.loadFromURL(domain)
+	}
+
+	// First try the well-known path.
+	wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
+
+	wellknownResult := pmdl.loadFromURL(wellknownURL)
+
+	// Valid provider metadata under well-known.
+	var wellknownGood *LoadedProviderMetadata
+
+	// We have a candidate.
+	if wellknownResult.Valid() {
+		wellknownGood = wellknownResult
+	}
+
+	// Next load the PMDs from security.txt
+	secURL := "https://" + domain + "/.well-known/security.txt"
+	secResults := pmdl.loadFromSecurity(secURL)
+
+	// Filter out the results which are valid.
+	var secGoods []*LoadedProviderMetadata
+
+	for _, result := range secResults {
+		if len(result.Messages) > 0 {
+			// If there where validation issues append them
+			// to the overall report
+			pmdl.messages.AppendUnique(pmdl.messages)
+		} else {
+			secGoods = append(secGoods, result)
+		}
+	}
+
+	// Mention extra CSAF entries in security.txt.
+	ignoreExtras := func() {
+		for _, extra := range secGoods[1:] {
+			pmdl.messages.Add(
+				ExtraProviderMetadataFound,
+				fmt.Sprintf("Ignoring extra CSAF entry in security.txt: %s", extra.URL))
+		}
+	}
+
+	// security.txt contains good entries.
+	if len(secGoods) > 0 {
+		// we already have a good wellknown, take it.
+		if wellknownGood != nil {
+			// check if first of security urls is identical to wellknown.
+			if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) {
+				ignoreExtras()
+			} else {
+				// Complaint about not matching.
+				pmdl.messages.Add(
+					WellknownSecurityMismatch,
+					"First entry of security.txt and well-known don't match.")
+				// List all the security urls.
+				for _, sec := range secGoods {
+					pmdl.messages.Add(
+						IgnoreProviderMetadata,
+						fmt.Sprintf("Ignoring CSAF entry in security.txt: %s", sec.URL))
+				}
+			}
+			// Take the good well-known.
+			wellknownGood.Messages.AppendUnique(pmdl.messages)
+			return wellknownGood
+		}
+
+		// Don't have well-known. Take first good from security.txt.
+		ignoreExtras()
+		secGoods[0].Messages.AppendUnique(pmdl.messages)
+		return secGoods[0]
+	}
+
+	// If we have a good well-known take it.
+	if wellknownGood != nil {
+		wellknownGood.Messages.AppendUnique(pmdl.messages)
+		return wellknownGood
+	}
+
+	// Last resort: fall back to DNS.
+	dnsURL := "https://csaf.data.security." + domain
+	return pmdl.loadFromURL(dnsURL)
+}
+
+// loadFromSecurity loads the PMDs mentioned in the security.txt.
+func (pmdl *ProviderMetadataLoader) loadFromSecurity(path string) []*LoadedProviderMetadata {
+
+	res, err := pmdl.client.Get(path)
+	if err != nil {
+		pmdl.messages.Add(
+			HTTPFailed,
+			fmt.Sprintf("Fetching %q failed: %v", path, err))
+		return nil
+	}
+	if res.StatusCode != http.StatusOK {
+		pmdl.messages.Add(
+			HTTPFailed,
+			fmt.Sprintf("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode))
+		return nil
+	}
+
+	// Extract all potential URLs from CSAF.
+	urls, err := func() ([]string, error) {
+		defer res.Body.Close()
+		return ExtractProviderURL(res.Body, true)
+	}()
+
+	if err != nil {
+		pmdl.messages.Add(
+			HTTPFailed,
+			fmt.Sprintf("Loading %q failed: %v", path, err))
+		return nil
+	}
+
+	var loaded []*LoadedProviderMetadata
+
+	// Load the URLs
+nextURL:
+	for _, url := range urls {
+		lpmd := pmdl.loadFromURL(url)
+		// If loading failed note it down.
+		if !lpmd.Valid() {
+			pmdl.messages.AppendUnique(lpmd.Messages)
+			continue
+		}
+		// Check for duplicates
+		for _, l := range loaded {
+			if l == lpmd {
+				continue nextURL
+			}
+		}
+		loaded = append(loaded, lpmd)
+	}
+
+	return loaded
+}
+
+// loadFromURL loads a provider metadata from a given URL.
+func (pmdl *ProviderMetadataLoader) loadFromURL(path string) *LoadedProviderMetadata {
+
+	result := LoadedProviderMetadata{URL: path}
+
+	res, err := pmdl.client.Get(path)
+	if err != nil {
+		result.Messages.Add(
+			HTTPFailed,
+			fmt.Sprintf("fetching %q failed: %v", path, err))
+		return &result
+	}
+	if res.StatusCode != http.StatusOK {
+		result.Messages.Add(
+			HTTPFailed,
+			fmt.Sprintf("fetching %q failed: %s (%d)", path, res.Status, res.StatusCode))
+		return &result
+	}
+
+	// TODO: Check for application/json and log it.
+
+	defer res.Body.Close()
+
+	// Calculate checksum for later comparison.
+	hash := sha256.New()
+
+	tee := io.TeeReader(res.Body, hash)
+
+	var doc any
+
+	if err := json.NewDecoder(tee).Decode(&doc); err != nil {
+		result.Messages.Add(
+			JSONDecodingFailed,
+			fmt.Sprintf("JSON decoding failed: %v", err))
+		return &result
+	}
+
+	// Before checking the err lets check if we had the same
+	// document before. If so it will have failed parsing before.
+
+	sum := hash.Sum(nil)
+	key := string(sum)
+
+	// If we already have loaded it return the cached result.
+	if r := pmdl.already[key]; r != nil {
+		return r
+	}
+
+	// write it back as loaded
+
+	switch errors, err := ValidateProviderMetadata(doc); {
+	case err != nil:
+		result.Messages.Add(
+			SchemaValidationFailed,
+			fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err))
+
+	case len(errors) > 0:
+		result.Messages = []ProviderMetadataLoadMessage{{
+			Type:    SchemaValidationFailed,
+			Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err),
+		}}
+		for _, msg := range errors {
+			result.Messages.Add(
+				SchemaValidationFailedDetail,
+				strings.ReplaceAll(msg, `%`, `%%`))
+		}
+	default:
+		// Only store in result if validation passed.
+		result.Document = doc
+		result.Hash = sum
+	}
+
+	pmdl.already[key] = &result
+	return &result
+}

csaf/util.go

@@ -10,299 +10,10 @@ package csaf
 
 import (
 	"bufio"
-	"bytes"
-	"crypto/sha256"
-	"encoding/json"
-	"fmt"
 	"io"
-	"log"
-	"net/http"
 	"strings"
-
-	"github.com/csaf-poc/csaf_distribution/util"
 )
 
-// LoadedProviderMetadata represents a loaded provider metadata.
-type LoadedProviderMetadata struct {
-	// URL is location where the document was found.
-	URL string
-	// Document is the de-serialized JSON document.
-	Document any
-	// Hash is a SHA256 sum over the document.
-	Hash []byte
-	// Messages are the error message happened while loading.
-	Messages []string
-}
-
-// Valid returns true if the loaded document is valid.
-func (lpm *LoadedProviderMetadata) Valid() bool {
-	return lpm != nil && lpm.Document != nil && lpm.Hash != nil
-}
-
-// defaultLogging generates a logging function if given is nil.
-func defaultLogging(
-	logging func(format string, args ...any),
-	prefix, suffix string,
-) func(format string, args ...any) {
-
-	if logging != nil {
-		return logging
-	}
-	return func(format string, args ...any) {
-		log.Printf(prefix+format+suffix, args...)
-	}
-}
-
-// LoadProviderMetadataFromURL loads a provider metadata from a given URL.
-// Returns nil if the document was not found.
-func LoadProviderMetadataFromURL(
-	client util.Client,
-	url string,
-	already map[string]*LoadedProviderMetadata,
-	logging func(format string, args ...any),
-) *LoadedProviderMetadata {
-
-	logging = defaultLogging(logging, "LoadProviderMetadataFromURL: ", "\n")
-
-	res, err := client.Get(url)
-	if err != nil {
-		logging("Fetching %q failed: %v", url, err)
-		return nil
-	}
-	if res.StatusCode != http.StatusOK {
-		logging("Fetching %q failed: %s (%d)", url, res.Status, res.StatusCode)
-		return nil
-	}
-
-	// TODO: Check for application/json and log it.
-
-	defer res.Body.Close()
-
-	// Calculate checksum for later comparison.
-	hash := sha256.New()
-
-	result := LoadedProviderMetadata{URL: url}
-
-	tee := io.TeeReader(res.Body, hash)
-
-	var doc any
-
-	err = json.NewDecoder(tee).Decode(&doc)
-	// Before checking the err lets check if we had the same
-	// document before. If so it will have failed parsing before.
-
-	sum := hash.Sum(nil)
-
-	var key string
-	if already != nil {
-		key = string(sum)
-		if r, ok := already[key]; ok {
-			return r
-		}
-	}
-
-	// write it back as loaded
-	storeLoaded := func() {
-		if already != nil {
-			already[key] = &result
-		}
-	}
-
-	// We have loaded it the first time.
-	if err != nil {
-		result.Messages = []string{fmt.Sprintf("%s: Decoding JSON failed: %v", url, err)}
-		storeLoaded()
-		return &result
-	}
-
-	switch errors, err := ValidateProviderMetadata(doc); {
-	case err != nil:
-		result.Messages = []string{
-			fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)}
-
-	case len(errors) > 0:
-		result.Messages = []string{
-			fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)}
-		for _, msg := range errors {
-			result.Messages = append(result.Messages, strings.ReplaceAll(msg, `%`, `%%`))
-		}
-	default:
-		// Only store in result if validation passed.
-		result.Document = doc
-		result.Hash = sum
-	}
-
-	storeLoaded()
-	return &result
-}
-
-// LoadProviderMetadatasFromSecurity loads a secturity.txt,
-// extracts and the CSAF urls from the document.
-// Returns nil if no url was successfully found.
-func LoadProviderMetadatasFromSecurity(
-	client util.Client,
-	path string,
-	already map[string]*LoadedProviderMetadata,
-	logging func(format string, args ...any),
-) []*LoadedProviderMetadata {
-
-	logging = defaultLogging(logging, "LoadProviderMetadataFromSecurity: ", "\n")
-
-	res, err := client.Get(path)
-	if err != nil {
-		logging("Fetching %q failed: %v", path, err)
-		return nil
-	}
-	if res.StatusCode != http.StatusOK {
-		logging("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode)
-		return nil
-	}
-
-	// Extract all potential URLs from CSAF.
-	urls, err := func() ([]string, error) {
-		defer res.Body.Close()
-		return ExtractProviderURL(res.Body, true)
-	}()
-
-	if err != nil {
-		// Treat as not found
-		return nil
-	}
-
-	var results []*LoadedProviderMetadata
-
-	// Load the URLs
-	for _, url := range urls {
-		if result := LoadProviderMetadataFromURL(
-			client, url, already, logging,
-		); result.Valid() {
-			results = append(results, result)
-		}
-	}
-
-	return results
-}
-
-// LoadProviderMetadataForDomain loads a provider metadata for a given domain.
-// Returns nil if no provider metadata (PMD) was found.
-// If the domain starts with `https://` it only attemps to load
-// the data from that URL.
-// The logging can be used to track the errors happening while loading.
-func LoadProviderMetadataForDomain(
-	client util.Client,
-	domain string,
-	logging func(format string, args ...any),
-) *LoadedProviderMetadata {
-
-	logging = defaultLogging(logging, "LoadProviderMetadataForDomain: ", "\n")
-
-	// As many URLs may lead to the same content only log once per content.
-	alreadyLogged := map[*LoadedProviderMetadata]string{}
-
-	lg := func(result *LoadedProviderMetadata, url string) {
-		if result == nil {
-			logging("%q not found.", url)
-			return
-		}
-		if other := alreadyLogged[result]; other != "" {
-			logging("%q is same %q.", url, other)
-			return
-		}
-		alreadyLogged[result] = url
-		for _, msg := range result.Messages {
-			logging(msg)
-		}
-	}
-
-	// keey track of already loaded pmds.
-	already := map[string]*LoadedProviderMetadata{}
-
-	// check direct path
-	if strings.HasPrefix(domain, "https://") {
-		result := LoadProviderMetadataFromURL(
-			client, domain, already, logging)
-		lg(result, domain)
-		return result
-	}
-
-	// Valid provider metadata under well-known.
-	var wellknownGood *LoadedProviderMetadata
-
-	// First try the well-known path.
-	wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
-	wellknownResult := LoadProviderMetadataFromURL(
-		client, wellknownURL, already, logging)
-	lg(wellknownResult, wellknownURL)
-
-	// We have a candidate.
-	if wellknownResult.Valid() {
-		wellknownGood = wellknownResult
-	}
-
-	// Next load the PMDs from security.txt
-	secURL := "https://" + domain + "/.well-known/security.txt"
-	secResults := LoadProviderMetadatasFromSecurity(
-		client, secURL, already, logging)
-
-	if len(secResults) == 0 {
-		logging("%s failed to load.", secURL)
-	} else {
-		// Filter out the results which are valid.
-		var secGoods []*LoadedProviderMetadata
-
-		for _, result := range secResults {
-			if len(result.Messages) > 0 {
-				lg(result, result.URL)
-			} else {
-				secGoods = append(secGoods, result)
-			}
-		}
-
-		// security.txt contains good entries.
-		if len(secGoods) > 0 {
-			// we already have a good wellknown, take it.
-			if wellknownGood != nil {
-				// check if first of security urls is identical to wellknown.
-				if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) {
-					// Mention extra CSAF entries
-					for _, extra := range secGoods[1:] {
-						logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL)
-					}
-				} else {
-					// Complaint about not matching.
-					logging("First entry of security.txt and well-known don't match.")
-					// List all the security urls.
-					for _, sec := range secGoods {
-						logging("Ignoring CSAF entry in security.txt: %s", sec.URL)
-					}
-				}
-				// Take the good well-known.
-				return wellknownGood
-			}
-
-			// Don't have well-known. Take first good from security.txt.
-			// Mention extra CSAF entries
-			for _, extra := range secGoods[1:] {
-				logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL)
-			}
-
-			return secGoods[0]
-		}
-	}
-
-	// If we have a good well-known take it.
-	if wellknownGood != nil {
-		return wellknownGood
-	}
-
-	// Last resort: fall back to DNS.
-	dnsURL := "https://csaf.data.security." + domain
-	dnsResult := LoadProviderMetadataFromURL(
-		client, dnsURL, already, logging)
-	lg(dnsResult, dnsURL)
-	return dnsResult
-}
-
 // ExtractProviderURL extracts URLs of provider metadata.
 // If all is true all URLs are returned. Otherwise only the first is returned.
 func ExtractProviderURL(r io.Reader, all bool) ([]string, error) {