diff --git a/cmd/csaf_aggregator/processor.go b/cmd/csaf_aggregator/processor.go index b665a0b..7ba511b 100644 --- a/cmd/csaf_aggregator/processor.go +++ b/cmd/csaf_aggregator/processor.go @@ -79,11 +79,17 @@ func (w *worker) createDir() (string, error) { func (w *worker) locateProviderMetadata(domain string) error { - lpmd := csaf.LoadProviderMetadataForDomain( - w.client, domain, func(format string, args ...any) { + loader := csaf.NewProviderMetadataLoader(w.client) + + lpmd := loader.Load(domain) + + if w.processor.cfg.Verbose { + for i := range lpmd.Messages { log.Printf( - "Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...) - }) + "Loading provider-metadata.json of %q: %s\n", + domain, lpmd.Messages[i].Message) + } + } if !lpmd.Valid() { return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 645a17a..a47b584 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -140,28 +140,6 @@ func writeReport(report *Report, opts *options) error { return writer(report, w) } -// buildReporters initializes each report by assigning a number and description to it. -// It returns an array of the reporter interface type. -func buildReporters() []reporter { - return []reporter{ - &validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}}, - &filenameReporter{baseReporter{num: 2, description: "Filename"}}, - &tlsReporter{baseReporter{num: 3, description: "TLS"}}, - &redirectsReporter{baseReporter{num: 6, description: "Redirects"}}, - &providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}}, - &securityReporter{baseReporter{num: 8, description: "security.txt"}}, - &wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}}, - &dnsPathReporter{baseReporter{num: 10, description: "DNS path"}}, - &oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}}, - &indexReporter{baseReporter{num: 12, description: "index.txt"}}, - &changesReporter{baseReporter{num: 13, description: "changes.csv"}}, - &directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}}, - &integrityReporter{baseReporter{num: 18, description: "Integrity"}}, - &signaturesReporter{baseReporter{num: 19, description: "Signatures"}}, - &publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}}, - } -} - // run uses a processor to check all the given domains or direct urls // and generates a report. func run(opts *options, domains []string) (*Report, error) { @@ -170,7 +148,7 @@ func run(opts *options, domains []string) (*Report, error) { return nil, err } defer p.close() - return p.run(buildReporters(), domains) + return p.run(domains) } func main() { diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 0613c33..3697c03 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -223,7 +223,7 @@ func (p *processor) clean() { // run calls checkDomain function for each domain in the given "domains" parameter. // Then it calls the report method on each report from the given "reporters" parameter for each domain. // It returns a pointer to the report and nil, otherwise an error. -func (p *processor) run(reporters []reporter, domains []string) (*Report, error) { +func (p *processor) run(domains []string) (*Report, error) { report := Report{ Date: ReportTime{Time: time.Now().UTC()}, @@ -231,19 +231,24 @@ func (p *processor) run(reporters []reporter, domains []string) (*Report, error) } for _, d := range domains { - if err := p.checkDomain(d); err != nil { - if err == errContinue || err == errStop { - continue + if p.checkProviderMetadata(d) { + if err := p.checkDomain(d); err != nil { + if err == errContinue || err == errStop { + continue + } + return nil, err } - return nil, err } domain := &Domain{Name: d} - for _, r := range reporters { - r.report(p, domain) - } if err := p.fillMeta(domain); err != nil { log.Printf("Filling meta data failed: %v\n", err) + // reporters depend on role. + continue + } + + for _, r := range buildReporters(*domain.Role) { + r.report(p, domain) } report.Domains = append(report.Domains, domain) @@ -287,7 +292,6 @@ func (p *processor) domainChecks(domain string) []func(*processor, string) error direct := strings.HasPrefix(domain, "https://") checks := []func(*processor, string) error{ - (*processor).checkProviderMetadata, (*processor).checkPGPKeys, } @@ -1113,26 +1117,32 @@ func (p *processor) checkListing(string) error { // decodes, and validates against the JSON schema. // According to the result, the respective error messages added to // badProviderMetadata. -// It returns nil if all checks are passed. -func (p *processor) checkProviderMetadata(domain string) error { +func (p *processor) checkProviderMetadata(domain string) bool { p.badProviderMetadata.use() client := p.httpClient() - lpmd := csaf.LoadProviderMetadataForDomain(client, domain, p.badProviderMetadata.warn) + loader := csaf.NewProviderMetadataLoader(client) + + lpmd := loader.Load(domain) + + for i := range lpmd.Messages { + // TODO: Filter depending on the role. + p.badProviderMetadata.error(lpmd.Messages[i].Message) + } if !lpmd.Valid() { p.badProviderMetadata.error("No valid provider-metadata.json found.") p.badProviderMetadata.error("STOPPING here - cannot perform other checks.") - return errStop + return false } p.pmdURL = lpmd.URL p.pmd256 = lpmd.Hash p.pmd = lpmd.Document - return nil + return true } // checkSecurity checks the security.txt file by making HTTP request to fetch it. diff --git a/cmd/csaf_checker/reporters.go b/cmd/csaf_checker/reporters.go index 19772a8..fc15f70 100644 --- a/cmd/csaf_checker/reporters.go +++ b/cmd/csaf_checker/reporters.go @@ -12,6 +12,8 @@ import ( "fmt" "sort" "strings" + + "github.com/csaf-poc/csaf_distribution/csaf" ) type ( @@ -22,6 +24,8 @@ type ( validReporter struct{ baseReporter } filenameReporter struct{ baseReporter } tlsReporter struct{ baseReporter } + tlpWhiteReporter struct{ baseReporter } + tlpAmberRedReporter struct{ baseReporter } redirectsReporter struct{ baseReporter } providerMetadataReport struct{ baseReporter } securityReporter struct{ baseReporter } @@ -31,11 +35,83 @@ type ( indexReporter struct{ baseReporter } changesReporter struct{ baseReporter } directoryListingsReporter struct{ baseReporter } + rolieFeedReporter struct{ baseReporter } + rolieServiceReporter struct{ baseReporter } + rolieCategoryReporter struct{ baseReporter } integrityReporter struct{ baseReporter } signaturesReporter struct{ baseReporter } publicPGPKeyReporter struct{ baseReporter } + listReporter struct{ baseReporter } + hasTwoReporter struct{ baseReporter } + mirrorReporter struct{ baseReporter } ) +var reporters = [23]reporter{ + &validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}}, + &filenameReporter{baseReporter{num: 2, description: "Filename"}}, + &tlsReporter{baseReporter{num: 3, description: "TLS"}}, + &tlpWhiteReporter{baseReporter{num: 4, description: "TLP:WHITE"}}, + &tlpAmberRedReporter{baseReporter{num: 5, description: "TLP:AMBER and TLP:RED"}}, + &redirectsReporter{baseReporter{num: 6, description: "Redirects"}}, + &providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}}, + &securityReporter{baseReporter{num: 8, description: "security.txt"}}, + &wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}}, + &dnsPathReporter{baseReporter{num: 10, description: "DNS path"}}, + &oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}}, + &indexReporter{baseReporter{num: 12, description: "index.txt"}}, + &changesReporter{baseReporter{num: 13, description: "changes.csv"}}, + &directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}}, + &rolieFeedReporter{baseReporter{num: 15, description: "ROLIE feed"}}, + &rolieServiceReporter{baseReporter{num: 16, description: "ROLIE service document"}}, + &rolieCategoryReporter{baseReporter{num: 17, description: "ROLIE category document"}}, + &integrityReporter{baseReporter{num: 18, description: "Integrity"}}, + &signaturesReporter{baseReporter{num: 19, description: "Signatures"}}, + &publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}}, + &listReporter{baseReporter{num: 21, description: "List of CSAF providers"}}, + &hasTwoReporter{baseReporter{num: 22, description: "Two disjoint issuing parties"}}, + &mirrorReporter{baseReporter{num: 23, description: "Mirror"}}, +} + +var roleImplies = map[csaf.MetadataRole][]csaf.MetadataRole{ + csaf.MetadataRoleProvider: {csaf.MetadataRolePublisher}, + csaf.MetadataRoleTrustedProvider: {csaf.MetadataRoleProvider}, +} + +func requirements(role csaf.MetadataRole) [][2]int { + var own [][2]int + switch role { + case csaf.MetadataRoleTrustedProvider: + own = [][2]int{{18, 20}} + case csaf.MetadataRoleProvider: + // TODO: use commented numbers when TLPs should be checked. + own = [][2]int{{6 /* 5 */, 7}, {8, 10}, {11, 14}, {15, 17}} + case csaf.MetadataRolePublisher: + own = [][2]int{{1, 3 /* 4 */}} + } + for _, base := range roleImplies[role] { + own = append(own, requirements(base)...) + } + return own +} + +// buildReporters initializes each report by assigning a number and description to it. +// It returns an array of the reporter interface type. +func buildReporters(role csaf.MetadataRole) []reporter { + var reps []reporter + reqs := requirements(role) + // sort to have them ordered by there number. + sort.Slice(reqs, func(i, j int) bool { return reqs[i][0] < reqs[j][0] }) + for _, req := range reqs { + from, to := req[0]-1, req[1]-1 + for i := from; i <= to; i++ { + if rep := reporters[i]; rep != nil { + reps = append(reps, rep) + } + } + } + return reps +} + func (bc *baseReporter) requirement(domain *Domain) *Requirement { req := &Requirement{ Num: bc.num, @@ -115,6 +191,21 @@ func (r *tlsReporter) report(p *processor, domain *Domain) { req.message(ErrorType, urls...) } +// report tests if a document labeled TLP:WHITE +// is freely accessible and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *tlpWhiteReporter) report(_ *processor, _ *Domain) { + // TODO +} + +// report tests if a document labeled TLP:AMBER +// or TLP:RED is access protected +// and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *tlpAmberRedReporter) report(_ *processor, _ *Domain) { + // TODO +} + // report tests if redirects are used and sets the "message" field value // of the "Requirement" struct as a result of that. func (r *redirectsReporter) report(p *processor, domain *Domain) { @@ -269,6 +360,31 @@ func (r *directoryListingsReporter) report(p *processor, domain *Domain) { req.Messages = p.badDirListings } +// report checks whether there is only a single ROLIE feed for a +// given TLP level and whether any of the TLP levels +// TLP:WHITE, TLP:GREEN or unlabeled exists and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *rolieFeedReporter) report(_ *processor, _ *Domain) { + // TODO +} + +// report tests whether a ROLIE service document is used and if so, +// whether it is a [RFC8322] conform JSON file that lists the +// ROLIE feed documents and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *rolieServiceReporter) report(_ *processor, _ *Domain) { + // TODO +} + +// report tests whether a ROLIE category document is used and if so, +// whether it is a [RFC8322] conform JSON file and is used to dissect +// documents by certain criteria +// and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *rolieCategoryReporter) report(_ *processor, _ *Domain) { + // TODO +} + func (r *integrityReporter) report(p *processor, domain *Domain) { req := r.requirement(domain) if !p.badIntegrities.used() { @@ -306,3 +422,25 @@ func (r *publicPGPKeyReporter) report(p *processor, domain *Domain) { p.keys.CountEntities())) } } + +// report tests whether a CSAF aggregator JSON schema conform +// aggregator.json exists without being adjacent to a +// provider-metadata.json +func (r *listReporter) report(_ *processor, _ *Domain) { + // TODO +} + +// report tests whether the aggregator.json lists at least +// two disjoint issuing parties. TODO: reevaluate phrasing (Req 7.1.22) +func (r *hasTwoReporter) report(_ *processor, _ *Domain) { + // TODO +} + +// report tests whether the CSAF documents of each issuing mirrored party +// is in a different folder, which are adjacent to the aggregator.json and +// if the folder name is retrieved from the name of the issuing authority. +// It also tests whether each folder has a provider-metadata.json for their +// party and provides ROLIE feed documents. +func (r *mirrorReporter) report(_ *processor, _ *Domain) { + // TODO +} diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 1e8b62e..d72d37b 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -118,11 +118,16 @@ func (d *downloader) httpClient() util.Client { func (d *downloader) download(ctx context.Context, domain string) error { client := d.httpClient() - lpmd := csaf.LoadProviderMetadataForDomain( - client, domain, func(format string, args ...any) { - log.Printf( - "Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...) - }) + loader := csaf.NewProviderMetadataLoader(client) + + lpmd := loader.Load(domain) + + if d.opts.Verbose { + for i := range lpmd.Messages { + log.Printf("Loading provider-metadata.json for %q: %s\n", + domain, lpmd.Messages[i].Message) + } + } if !lpmd.Valid() { return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go new file mode 100644 index 0000000..7e333e6 --- /dev/null +++ b/csaf/providermetaloader.go @@ -0,0 +1,328 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +package csaf + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/csaf-poc/csaf_distribution/util" +) + +// ProviderMetadataLoader helps load provider-metadata.json from +// the various locations. +type ProviderMetadataLoader struct { + client util.Client + already map[string]*LoadedProviderMetadata + messages ProviderMetadataLoadMessages +} + +// ProviderMetadataLoadMessageType is the type of the message. +type ProviderMetadataLoadMessageType int + +const ( + //JSONDecodingFailed indicates problems with JSON decoding + JSONDecodingFailed ProviderMetadataLoadMessageType = iota + // SchemaValidationFailed indicates a general problem with schema validation. + SchemaValidationFailed + // SchemaValidationFailedDetail is a failure detail in schema validation. + SchemaValidationFailedDetail + // HTTPFailed indicates that loading on HTTP level failed. + HTTPFailed + // ExtraProviderMetadataFound indicates an extra PMD found in security.txt. + ExtraProviderMetadataFound + // WellknownSecurityMismatch indicates that the PMDs found under wellknown and + // in the security do not match. + WellknownSecurityMismatch + // IgnoreProviderMetadata indicates that a extra PMD was ignored. + IgnoreProviderMetadata +) + +// ProviderMetadataLoadMessage is a message generated while loading +// a provider meta data file. +type ProviderMetadataLoadMessage struct { + Type ProviderMetadataLoadMessageType + Message string +} + +// ProviderMetadataLoadMessages is a list of loading messages. +type ProviderMetadataLoadMessages []ProviderMetadataLoadMessage + +// LoadedProviderMetadata represents a loaded provider metadata. +type LoadedProviderMetadata struct { + // URL is location where the document was found. + URL string + // Document is the de-serialized JSON document. + Document any + // Hash is a SHA256 sum over the document. + Hash []byte + // Messages are the error message happened while loading. + Messages ProviderMetadataLoadMessages +} + +// Add appends a message to the list of loading messages. +func (pmlm *ProviderMetadataLoadMessages) Add( + typ ProviderMetadataLoadMessageType, + msg string, +) { + *pmlm = append(*pmlm, ProviderMetadataLoadMessage{ + Type: typ, + Message: msg, + }) +} + +// AppendUnique appends unique messages from a second list. +func (pmlm *ProviderMetadataLoadMessages) AppendUnique(other ProviderMetadataLoadMessages) { +next: + for _, o := range other { + for _, m := range *pmlm { + if m == o { + continue next + } + } + *pmlm = append(*pmlm, o) + } +} + +// Valid returns true if the loaded document is valid. +func (lpm *LoadedProviderMetadata) Valid() bool { + return lpm != nil && lpm.Document != nil && lpm.Hash != nil +} + +// NewProviderMetadataLoader create a new loader. +func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader { + return &ProviderMetadataLoader{ + client: client, + already: map[string]*LoadedProviderMetadata{}, + } +} + +// Load loads a provider metadata for a given path. +// If the domain starts with `https://` it only attemps to load +// the data from that URL. +func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata { + + // Check direct path + if strings.HasPrefix(domain, "https://") { + return pmdl.loadFromURL(domain) + } + + // First try the well-known path. + wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" + + wellknownResult := pmdl.loadFromURL(wellknownURL) + + // Valid provider metadata under well-known. + var wellknownGood *LoadedProviderMetadata + + // We have a candidate. + if wellknownResult.Valid() { + wellknownGood = wellknownResult + } + + // Next load the PMDs from security.txt + secURL := "https://" + domain + "/.well-known/security.txt" + secResults := pmdl.loadFromSecurity(secURL) + + // Filter out the results which are valid. + var secGoods []*LoadedProviderMetadata + + for _, result := range secResults { + if len(result.Messages) > 0 { + // If there where validation issues append them + // to the overall report + pmdl.messages.AppendUnique(pmdl.messages) + } else { + secGoods = append(secGoods, result) + } + } + + // Mention extra CSAF entries in security.txt. + ignoreExtras := func() { + for _, extra := range secGoods[1:] { + pmdl.messages.Add( + ExtraProviderMetadataFound, + fmt.Sprintf("Ignoring extra CSAF entry in security.txt: %s", extra.URL)) + } + } + + // security.txt contains good entries. + if len(secGoods) > 0 { + // we already have a good wellknown, take it. + if wellknownGood != nil { + // check if first of security urls is identical to wellknown. + if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) { + ignoreExtras() + } else { + // Complaint about not matching. + pmdl.messages.Add( + WellknownSecurityMismatch, + "First entry of security.txt and well-known don't match.") + // List all the security urls. + for _, sec := range secGoods { + pmdl.messages.Add( + IgnoreProviderMetadata, + fmt.Sprintf("Ignoring CSAF entry in security.txt: %s", sec.URL)) + } + } + // Take the good well-known. + wellknownGood.Messages.AppendUnique(pmdl.messages) + return wellknownGood + } + + // Don't have well-known. Take first good from security.txt. + ignoreExtras() + secGoods[0].Messages.AppendUnique(pmdl.messages) + return secGoods[0] + } + + // If we have a good well-known take it. + if wellknownGood != nil { + wellknownGood.Messages.AppendUnique(pmdl.messages) + return wellknownGood + } + + // Last resort: fall back to DNS. + dnsURL := "https://csaf.data.security." + domain + return pmdl.loadFromURL(dnsURL) +} + +// loadFromSecurity loads the PMDs mentioned in the security.txt. +func (pmdl *ProviderMetadataLoader) loadFromSecurity(path string) []*LoadedProviderMetadata { + + res, err := pmdl.client.Get(path) + if err != nil { + pmdl.messages.Add( + HTTPFailed, + fmt.Sprintf("Fetching %q failed: %v", path, err)) + return nil + } + if res.StatusCode != http.StatusOK { + pmdl.messages.Add( + HTTPFailed, + fmt.Sprintf("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode)) + return nil + } + + // Extract all potential URLs from CSAF. + urls, err := func() ([]string, error) { + defer res.Body.Close() + return ExtractProviderURL(res.Body, true) + }() + + if err != nil { + pmdl.messages.Add( + HTTPFailed, + fmt.Sprintf("Loading %q failed: %v", path, err)) + return nil + } + + var loaded []*LoadedProviderMetadata + + // Load the URLs +nextURL: + for _, url := range urls { + lpmd := pmdl.loadFromURL(url) + // If loading failed note it down. + if !lpmd.Valid() { + pmdl.messages.AppendUnique(lpmd.Messages) + continue + } + // Check for duplicates + for _, l := range loaded { + if l == lpmd { + continue nextURL + } + } + loaded = append(loaded, lpmd) + } + + return loaded +} + +// loadFromURL loads a provider metadata from a given URL. +func (pmdl *ProviderMetadataLoader) loadFromURL(path string) *LoadedProviderMetadata { + + result := LoadedProviderMetadata{URL: path} + + res, err := pmdl.client.Get(path) + if err != nil { + result.Messages.Add( + HTTPFailed, + fmt.Sprintf("fetching %q failed: %v", path, err)) + return &result + } + if res.StatusCode != http.StatusOK { + result.Messages.Add( + HTTPFailed, + fmt.Sprintf("fetching %q failed: %s (%d)", path, res.Status, res.StatusCode)) + return &result + } + + // TODO: Check for application/json and log it. + + defer res.Body.Close() + + // Calculate checksum for later comparison. + hash := sha256.New() + + tee := io.TeeReader(res.Body, hash) + + var doc any + + if err := json.NewDecoder(tee).Decode(&doc); err != nil { + result.Messages.Add( + JSONDecodingFailed, + fmt.Sprintf("JSON decoding failed: %v", err)) + return &result + } + + // Before checking the err lets check if we had the same + // document before. If so it will have failed parsing before. + + sum := hash.Sum(nil) + key := string(sum) + + // If we already have loaded it return the cached result. + if r := pmdl.already[key]; r != nil { + return r + } + + // write it back as loaded + + switch errors, err := ValidateProviderMetadata(doc); { + case err != nil: + result.Messages.Add( + SchemaValidationFailed, + fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err)) + + case len(errors) > 0: + result.Messages = []ProviderMetadataLoadMessage{{ + Type: SchemaValidationFailed, + Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err), + }} + for _, msg := range errors { + result.Messages.Add( + SchemaValidationFailedDetail, + strings.ReplaceAll(msg, `%`, `%%`)) + } + default: + // Only store in result if validation passed. + result.Document = doc + result.Hash = sum + } + + pmdl.already[key] = &result + return &result +} diff --git a/csaf/util.go b/csaf/util.go index ea3faee..f192f09 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -10,299 +10,10 @@ package csaf import ( "bufio" - "bytes" - "crypto/sha256" - "encoding/json" - "fmt" "io" - "log" - "net/http" "strings" - - "github.com/csaf-poc/csaf_distribution/util" ) -// LoadedProviderMetadata represents a loaded provider metadata. -type LoadedProviderMetadata struct { - // URL is location where the document was found. - URL string - // Document is the de-serialized JSON document. - Document any - // Hash is a SHA256 sum over the document. - Hash []byte - // Messages are the error message happened while loading. - Messages []string -} - -// Valid returns true if the loaded document is valid. -func (lpm *LoadedProviderMetadata) Valid() bool { - return lpm != nil && lpm.Document != nil && lpm.Hash != nil -} - -// defaultLogging generates a logging function if given is nil. -func defaultLogging( - logging func(format string, args ...any), - prefix, suffix string, -) func(format string, args ...any) { - - if logging != nil { - return logging - } - return func(format string, args ...any) { - log.Printf(prefix+format+suffix, args...) - } -} - -// LoadProviderMetadataFromURL loads a provider metadata from a given URL. -// Returns nil if the document was not found. -func LoadProviderMetadataFromURL( - client util.Client, - url string, - already map[string]*LoadedProviderMetadata, - logging func(format string, args ...any), -) *LoadedProviderMetadata { - - logging = defaultLogging(logging, "LoadProviderMetadataFromURL: ", "\n") - - res, err := client.Get(url) - if err != nil { - logging("Fetching %q failed: %v", url, err) - return nil - } - if res.StatusCode != http.StatusOK { - logging("Fetching %q failed: %s (%d)", url, res.Status, res.StatusCode) - return nil - } - - // TODO: Check for application/json and log it. - - defer res.Body.Close() - - // Calculate checksum for later comparison. - hash := sha256.New() - - result := LoadedProviderMetadata{URL: url} - - tee := io.TeeReader(res.Body, hash) - - var doc any - - err = json.NewDecoder(tee).Decode(&doc) - // Before checking the err lets check if we had the same - // document before. If so it will have failed parsing before. - - sum := hash.Sum(nil) - - var key string - if already != nil { - key = string(sum) - if r, ok := already[key]; ok { - return r - } - } - - // write it back as loaded - storeLoaded := func() { - if already != nil { - already[key] = &result - } - } - - // We have loaded it the first time. - if err != nil { - result.Messages = []string{fmt.Sprintf("%s: Decoding JSON failed: %v", url, err)} - storeLoaded() - return &result - } - - switch errors, err := ValidateProviderMetadata(doc); { - case err != nil: - result.Messages = []string{ - fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)} - - case len(errors) > 0: - result.Messages = []string{ - fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)} - for _, msg := range errors { - result.Messages = append(result.Messages, strings.ReplaceAll(msg, `%`, `%%`)) - } - default: - // Only store in result if validation passed. - result.Document = doc - result.Hash = sum - } - - storeLoaded() - return &result -} - -// LoadProviderMetadatasFromSecurity loads a secturity.txt, -// extracts and the CSAF urls from the document. -// Returns nil if no url was successfully found. -func LoadProviderMetadatasFromSecurity( - client util.Client, - path string, - already map[string]*LoadedProviderMetadata, - logging func(format string, args ...any), -) []*LoadedProviderMetadata { - - logging = defaultLogging(logging, "LoadProviderMetadataFromSecurity: ", "\n") - - res, err := client.Get(path) - if err != nil { - logging("Fetching %q failed: %v", path, err) - return nil - } - if res.StatusCode != http.StatusOK { - logging("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode) - return nil - } - - // Extract all potential URLs from CSAF. - urls, err := func() ([]string, error) { - defer res.Body.Close() - return ExtractProviderURL(res.Body, true) - }() - - if err != nil { - // Treat as not found - return nil - } - - var results []*LoadedProviderMetadata - - // Load the URLs - for _, url := range urls { - if result := LoadProviderMetadataFromURL( - client, url, already, logging, - ); result.Valid() { - results = append(results, result) - } - } - - return results -} - -// LoadProviderMetadataForDomain loads a provider metadata for a given domain. -// Returns nil if no provider metadata (PMD) was found. -// If the domain starts with `https://` it only attemps to load -// the data from that URL. -// The logging can be used to track the errors happening while loading. -func LoadProviderMetadataForDomain( - client util.Client, - domain string, - logging func(format string, args ...any), -) *LoadedProviderMetadata { - - logging = defaultLogging(logging, "LoadProviderMetadataForDomain: ", "\n") - - // As many URLs may lead to the same content only log once per content. - alreadyLogged := map[*LoadedProviderMetadata]string{} - - lg := func(result *LoadedProviderMetadata, url string) { - if result == nil { - logging("%q not found.", url) - return - } - if other := alreadyLogged[result]; other != "" { - logging("%q is same %q.", url, other) - return - } - alreadyLogged[result] = url - for _, msg := range result.Messages { - logging(msg) - } - } - - // keey track of already loaded pmds. - already := map[string]*LoadedProviderMetadata{} - - // check direct path - if strings.HasPrefix(domain, "https://") { - result := LoadProviderMetadataFromURL( - client, domain, already, logging) - lg(result, domain) - return result - } - - // Valid provider metadata under well-known. - var wellknownGood *LoadedProviderMetadata - - // First try the well-known path. - wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" - wellknownResult := LoadProviderMetadataFromURL( - client, wellknownURL, already, logging) - lg(wellknownResult, wellknownURL) - - // We have a candidate. - if wellknownResult.Valid() { - wellknownGood = wellknownResult - } - - // Next load the PMDs from security.txt - secURL := "https://" + domain + "/.well-known/security.txt" - secResults := LoadProviderMetadatasFromSecurity( - client, secURL, already, logging) - - if len(secResults) == 0 { - logging("%s failed to load.", secURL) - } else { - // Filter out the results which are valid. - var secGoods []*LoadedProviderMetadata - - for _, result := range secResults { - if len(result.Messages) > 0 { - lg(result, result.URL) - } else { - secGoods = append(secGoods, result) - } - } - - // security.txt contains good entries. - if len(secGoods) > 0 { - // we already have a good wellknown, take it. - if wellknownGood != nil { - // check if first of security urls is identical to wellknown. - if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) { - // Mention extra CSAF entries - for _, extra := range secGoods[1:] { - logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL) - } - } else { - // Complaint about not matching. - logging("First entry of security.txt and well-known don't match.") - // List all the security urls. - for _, sec := range secGoods { - logging("Ignoring CSAF entry in security.txt: %s", sec.URL) - } - } - // Take the good well-known. - return wellknownGood - } - - // Don't have well-known. Take first good from security.txt. - // Mention extra CSAF entries - for _, extra := range secGoods[1:] { - logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL) - } - - return secGoods[0] - } - } - - // If we have a good well-known take it. - if wellknownGood != nil { - return wellknownGood - } - - // Last resort: fall back to DNS. - dnsURL := "https://csaf.data.security." + domain - dnsResult := LoadProviderMetadataFromURL( - client, dnsURL, already, logging) - lg(dnsResult, dnsURL) - return dnsResult -} - // ExtractProviderURL extracts URLs of provider metadata. // If all is true all URLs are returned. Otherwise only the first is returned. func ExtractProviderURL(r io.Reader, all bool) ([]string, error) {