From 527a6f60051df1798e3dea23c3c6733fefe5bb69 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 31 May 2022 18:10:18 +0200 Subject: [PATCH 1/2] Implement better search for provider-metadata.json * Decouple loading of provider metadata from processor and moved in the base library. * Integrate new code into checker and aggregator * Adhere to csd02 revision of CSAF 2.0. resolve #60 --- cmd/csaf_aggregator/processor.go | 80 ++---------- cmd/csaf_checker/processor.go | 147 ++------------------- csaf/util.go | 216 +++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+), 207 deletions(-) diff --git a/cmd/csaf_aggregator/processor.go b/cmd/csaf_aggregator/processor.go index e5c8b52..a7dbfc4 100644 --- a/cmd/csaf_aggregator/processor.go +++ b/cmd/csaf_aggregator/processor.go @@ -9,14 +9,10 @@ package main import ( - "encoding/json" - "errors" - "io" + "fmt" "log" - "net/http" "os" "path/filepath" - "strings" "github.com/ProtonMail/gopenpgp/v2/crypto" "github.com/csaf-poc/csaf_distribution/csaf" @@ -75,76 +71,22 @@ func (w *worker) createDir() (string, error) { return dir, err } -// httpsDomain prefixes a domain with 'https://'. -func httpsDomain(domain string) string { - if strings.HasPrefix(domain, "https://") { - return domain - } - return "https://" + domain -} - -var providerMetadataLocations = [...]string{ - ".well-known/csaf", - "security/data/csaf", - "advisories/csaf", - "security/csaf", -} - func (w *worker) locateProviderMetadata(domain string) error { - w.metadataProvider = nil + lpmd := csaf.LoadProviderMetadataForDomain( + w.client, domain, func(format string, args ...interface{}) { + log.Printf( + "Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...) + }) - download := func(r io.Reader) error { - if err := json.NewDecoder(r).Decode(&w.metadataProvider); err != nil { - log.Printf("error: %s\n", err) - return errNotFound - } - return nil + if lpmd == nil { + return fmt.Errorf("no provider-metadata.json found for '%s'", domain) } - hd := httpsDomain(domain) - for _, loc := range providerMetadataLocations { - url := hd + "/" + loc - if err := downloadJSON(w.client, url, download); err != nil { - if err == errNotFound { - continue - } - return err - } - if w.metadataProvider != nil { - w.loc = loc - return nil - } - } + w.metadataProvider = lpmd.Document + w.loc = lpmd.URL - // Read from security.txt - - path := hd + "/.well-known/security.txt" - res, err := w.client.Get(path) - if err != nil { - return err - } - - if res.StatusCode != http.StatusOK { - return errNotFound - } - - if err := func() error { - defer res.Body.Close() - urls, err := csaf.ExtractProviderURL(res.Body, false) - if err != nil { - return err - } - if len(urls) == 0 { - return errors.New("no provider-metadata.json found in secturity.txt") - } - w.loc = urls[0] - return nil - }(); err != nil { - return err - } - - return downloadJSON(w.client, w.loc, download) + return nil } // removeOrphans removes the directories that are not in the providers list. diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index c5c4480..3ac3ff4 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -795,111 +795,6 @@ func (p *processor) checkListing(string) error { return nil } -var providerMetadataLocations = [...]string{ - ".well-known/csaf", - "security/data/csaf", - "advisories/csaf", - "security/csaf", -} - -// locateProviderMetadata searches for provider-metadata.json at various -// locations mentioned in "7.1.7 Requirement 7: provider-metadata.json". -func (p *processor) locateProviderMetadata( - domain string, - found func(string, io.Reader) error, -) error { - - client := p.httpClient() - tryURL := func(url string) (bool, error) { - log.Printf("Trying: %v\n", url) - res, err := client.Get(url) - - if err != nil || res.StatusCode != http.StatusOK || - res.Header.Get("Content-Type") != "application/json" { - // ignore this as it is expected. - return false, nil - } - - if err := func() error { - defer res.Body.Close() - return found(url, res.Body) - }(); err != nil { - return false, err - } - return true, nil - } - - for _, loc := range providerMetadataLocations { - url := "https://" + domain + "/" + loc + "/provider-metadata.json" - ok, err := tryURL(url) - if err != nil { - if err == errContinue { - continue - } - return err - } - if ok { - return nil - } - } - - // Read from security.txt - - path := "https://" + domain + "/.well-known/security.txt" - log.Printf("Searching in: %v\n", path) - res, err := client.Get(path) - if err == nil && res.StatusCode == http.StatusOK { - loc, err := func() (string, error) { - defer res.Body.Close() - return p.extractProviderURL(res.Body) - }() - - if err != nil { - log.Printf("did not find provider URL in /.well-known/security.txt, error: %v\n", err) - } - - if loc != "" { - if _, err = tryURL(loc); err == errContinue { - err = nil - } - return err - } - } - - // Read from DNS path - - path = "https://csaf.data.security." + domain - ok, err := tryURL(path) - if err != nil { - return err - } - if ok { - return nil - } - - return errStop -} - -func (p *processor) extractProviderURL(r io.Reader) (string, error) { - urls, err := csaf.ExtractProviderURL(r, true) - if err != nil { - return "", err - } - if len(urls) == 0 { - return "", errors.New("no provider-metadata.json found") - } - - if len(urls) > 1 { - p.badSecurity.use() - p.badSecurity.add("Found %d CSAF entries in security.txt", len(urls)) - } - if !strings.HasPrefix(urls[0], "https://") { - p.badSecurity.use() - p.badSecurity.add("CSAF URL does not start with https://: %s", urls[0]) - } - return urls[0], nil -} - // checkProviderMetadata checks provider-metadata.json. If it exists, // decodes, and validates against the JSON schema. // According to the result, the respective error messages added to @@ -909,44 +804,20 @@ func (p *processor) checkProviderMetadata(domain string) error { p.badProviderMetadata.use() - found := func(url string, content io.Reader) error { + client := p.httpClient() - // Calculate checksum for later comparison. - hash := sha256.New() + lpmd := csaf.LoadProviderMetadataForDomain(client, domain, p.badProviderMetadata.add) - tee := io.TeeReader(content, hash) - if err := json.NewDecoder(tee).Decode(&p.pmd); err != nil { - p.badProviderMetadata.add("%s: Decoding JSON failed: %v", url, err) - return errContinue - } - - p.pmd256 = hash.Sum(nil) - - errors, err := csaf.ValidateProviderMetadata(p.pmd) - if err != nil { - return err - } - if len(errors) > 0 { - p.badProviderMetadata.add("%s: Validating against JSON schema failed:", url) - for _, msg := range errors { - p.badProviderMetadata.add(strings.ReplaceAll(msg, `%`, `%%`)) - } - p.badProviderMetadata.add("STOPPING here - cannot perform other checks.") - return errStop - } - p.pmdURL = url - return nil - } - - if err := p.locateProviderMetadata(domain, found); err != nil { - return err - } - - if p.pmdURL == "" { - p.badProviderMetadata.add("No provider-metadata.json found.") + if lpmd == nil { + p.badProviderMetadata.add("No valid provider-metadata.json found.") p.badProviderMetadata.add("STOPPING here - cannot perform other checks.") return errStop } + + p.pmdURL = lpmd.URL + p.pmd256 = lpmd.Hash + p.pmd = lpmd.Document + return nil } diff --git a/csaf/util.go b/csaf/util.go index f192f09..2e2ae14 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -10,10 +10,226 @@ package csaf import ( "bufio" + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" "io" + "log" + "net/http" "strings" + + "github.com/csaf-poc/csaf_distribution/util" ) +// LoadedProviderMetadata represents a loaded provider metadata. +type LoadedProviderMetadata struct { + // URL is location where the document was found. + URL string + // Document is the de-serialized JSON document. + Document interface{} + // Hash is a SHA256 sum over the document. + Hash []byte + // Messages are the error message happened while loading. + Messages []string +} + +// LoadProviderMetadataFromURL loads a provider metadata from a given URL. +// Returns nil if the document was not found. +func LoadProviderMetadataFromURL(client util.Client, url string) *LoadedProviderMetadata { + + res, err := client.Get(url) + + if err != nil || res.StatusCode != http.StatusOK { + // Treat as not found. + return nil + } + + // TODO: Check for application/json and log it. + + defer res.Body.Close() + + // Calculate checksum for later comparison. + hash := sha256.New() + + result := LoadedProviderMetadata{URL: url} + + tee := io.TeeReader(res.Body, hash) + + if err := json.NewDecoder(tee).Decode(&result.Document); err != nil { + result.Messages = []string{fmt.Sprintf("%s: Decoding JSON failed: %v", url, err)} + return &result + } + + result.Hash = hash.Sum(nil) + + errors, err := ValidateProviderMetadata(result.Document) + if err != nil { + result.Messages = []string{ + fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)} + return &result + } + + if len(errors) > 0 { + result.Messages = []string{ + fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)} + for _, msg := range errors { + result.Messages = append(result.Messages, strings.ReplaceAll(msg, `%`, `%%`)) + } + } + + return &result +} + +// LoadProviderMetadatasFromSecurity loads a secturity.txt, +// extracts and the CSAF urls from the document. +// Returns nil if no url was successfully found. +func LoadProviderMetadatasFromSecurity(client util.Client, path string) []*LoadedProviderMetadata { + + res, err := client.Get(path) + + if err != nil || res.StatusCode != http.StatusOK { + // Treat as not found. + return nil + } + + // Extract all potential URLs from CSAF. + urls, err := func() ([]string, error) { + defer res.Body.Close() + return ExtractProviderURL(res.Body, true) + }() + + if err != nil { + // Treat as not found + return nil + } + + var results []*LoadedProviderMetadata + + // Load the URLs + for _, url := range urls { + if result := LoadProviderMetadataFromURL(client, url); result != nil { + results = append(results, result) + } + } + + return results +} + +// LoadProviderMetadataForDomain loads a provider metadata for a given domain. +// Returns nil if no provider metadata was found. +// The logging can be use to track the errors happening while loading. +func LoadProviderMetadataForDomain( + client util.Client, + domain string, + logging func(format string, args ...interface{}), +) *LoadedProviderMetadata { + + if logging == nil { + logging = func(format string, args ...interface{}) { + log.Printf("FindProviderMetadata: "+format+"\n", args...) + } + } + + // Valid provider metadata under well-known. + var wellknownGood *LoadedProviderMetadata + + // First try well-know path + wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" + log.Printf("Trying: %s\n", wellknownURL) + wellknownResult := LoadProviderMetadataFromURL(client, wellknownURL) + + if wellknownResult == nil { + logging("%s not found.", wellknownURL) + } else if len(wellknownResult.Messages) > 0 { + // There are issues + for _, msg := range wellknownResult.Messages { + logging(msg) + } + } else { + // We have a candidate. + wellknownGood = wellknownResult + } + + // Next load the PMDs from security.txt + secURL := "https://" + domain + "/.well-known/security.txt" + log.Printf("Trying: %s\n", secURL) + secResults := LoadProviderMetadatasFromSecurity(client, secURL) + + if secResults == nil { + logging("%s failed to load.", secURL) + } else { + // Filter out the results which are valid. + var secGoods []*LoadedProviderMetadata + + for _, result := range secResults { + if len(result.Messages) > 0 { + for _, msg := range result.Messages { + logging(msg) + } + } else { + secGoods = append(secGoods, result) + } + } + + // security.txt contains good entries. + if len(secGoods) > 0 { + // we have a wellknown good take it. + if wellknownGood != nil { + // check if first of security urls is identical to wellknown. + if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) { + // Mention extra CSAF entries + for _, extra := range secGoods[1:] { + logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL) + } + } else { + // Complaint about not matching. + logging("First entry of security.txt and well-known don't match.") + // List all the security urls. + for _, sec := range secGoods { + logging("Ignoring CSAF entry in security.txt: %s", sec.URL) + } + } + // Take the good well-known. + return wellknownGood + } + + // Don't have well-known. Take first good from security.txt. + // Mention extra CSAF entries + for _, extra := range secGoods[1:] { + logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL) + } + + return secGoods[0] + } + } + + // If we have a good well-known take it. + if wellknownGood != nil { + return wellknownGood + } + + // Last resort fall back to DNS. + + dnsURL := "https://csaf.data.security." + domain + log.Printf("Trying: %s\n", dnsURL) + dnsResult := LoadProviderMetadataFromURL(client, dnsURL) + + if dnsResult == nil { + logging("%s not found.", dnsURL) + } else if len(dnsResult.Messages) > 0 { + for _, msg := range dnsResult.Messages { + logging(msg) + } + } else { + // DNS seems to be okay. + return dnsResult + } + + // We failed all. + return nil +} + // ExtractProviderURL extracts URLs of provider metadata. // If all is true all URLs are returned. Otherwise only the first is returned. func ExtractProviderURL(r io.Reader, all bool) ([]string, error) { From c0aa7edc7029fafb6783dd63cc596e176db99fa4 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 1 Jun 2022 09:15:31 +0200 Subject: [PATCH 2/2] Improve code style * Remove unnecessary brackets in logical comparison. --- cmd/csaf_checker/main.go | 2 +- cmd/csaf_uploader/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index e4dde11..09d69c8 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -146,7 +146,7 @@ func main() { return } - if (opts.ClientCert != nil && opts.ClientKey == nil) || (opts.ClientCert == nil && opts.ClientKey != nil) { + if opts.ClientCert != nil && opts.ClientKey == nil || opts.ClientCert == nil && opts.ClientKey != nil { log.Println("Both client-key and client-cert options must be set for the authentication.") return } diff --git a/cmd/csaf_uploader/main.go b/cmd/csaf_uploader/main.go index dbbb01b..9007f92 100644 --- a/cmd/csaf_uploader/main.go +++ b/cmd/csaf_uploader/main.go @@ -392,7 +392,7 @@ func main() { check(readInteractive("Enter OpenPGP passphrase: ", &opts.Passphrase)) } - if (opts.ClientCert != nil && opts.ClientKey == nil) || (opts.ClientCert == nil && opts.ClientKey != nil) { + if opts.ClientCert != nil && opts.ClientKey == nil || opts.ClientCert == nil && opts.ClientKey != nil { log.Println("Both client-key and client-cert options must be set for the authentication.") return }