From 3445e58e45dc90d87c94b7acd7e6f6e46ed0e9c6 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 24 Mar 2023 18:24:46 +0100 Subject: [PATCH 01/14] Unexport some symbols --- csaf/util.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/csaf/util.go b/csaf/util.go index ea3faee..1e7cf97 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -53,16 +53,16 @@ func defaultLogging( } } -// LoadProviderMetadataFromURL loads a provider metadata from a given URL. +// loadProviderMetadataFromURL loads a provider metadata from a given URL. // Returns nil if the document was not found. -func LoadProviderMetadataFromURL( +func loadProviderMetadataFromURL( client util.Client, url string, already map[string]*LoadedProviderMetadata, logging func(format string, args ...any), ) *LoadedProviderMetadata { - logging = defaultLogging(logging, "LoadProviderMetadataFromURL: ", "\n") + logging = defaultLogging(logging, "loadProviderMetadataFromURL: ", "\n") res, err := client.Get(url) if err != nil { @@ -136,17 +136,17 @@ func LoadProviderMetadataFromURL( return &result } -// LoadProviderMetadatasFromSecurity loads a secturity.txt, +// loadProviderMetadatasFromSecurity loads a secturity.txt, // extracts and the CSAF urls from the document. // Returns nil if no url was successfully found. -func LoadProviderMetadatasFromSecurity( +func loadProviderMetadatasFromSecurity( client util.Client, path string, already map[string]*LoadedProviderMetadata, logging func(format string, args ...any), ) []*LoadedProviderMetadata { - logging = defaultLogging(logging, "LoadProviderMetadataFromSecurity: ", "\n") + logging = defaultLogging(logging, "loadProviderMetadataFromSecurity: ", "\n") res, err := client.Get(path) if err != nil { @@ -173,7 +173,7 @@ func LoadProviderMetadatasFromSecurity( // Load the URLs for _, url := range urls { - if result := LoadProviderMetadataFromURL( + if result := loadProviderMetadataFromURL( client, url, already, logging, ); result.Valid() { results = append(results, result) @@ -219,7 +219,7 @@ func LoadProviderMetadataForDomain( // check direct path if strings.HasPrefix(domain, "https://") { - result := LoadProviderMetadataFromURL( + result := loadProviderMetadataFromURL( client, domain, already, logging) lg(result, domain) return result @@ -230,7 +230,7 @@ func LoadProviderMetadataForDomain( // First try the well-known path. wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" - wellknownResult := LoadProviderMetadataFromURL( + wellknownResult := loadProviderMetadataFromURL( client, wellknownURL, already, logging) lg(wellknownResult, wellknownURL) @@ -241,7 +241,7 @@ func LoadProviderMetadataForDomain( // Next load the PMDs from security.txt secURL := "https://" + domain + "/.well-known/security.txt" - secResults := LoadProviderMetadatasFromSecurity( + secResults := loadProviderMetadatasFromSecurity( client, secURL, already, logging) if len(secResults) == 0 { @@ -297,7 +297,7 @@ func LoadProviderMetadataForDomain( // Last resort: fall back to DNS. dnsURL := "https://csaf.data.security." + domain - dnsResult := LoadProviderMetadataFromURL( + dnsResult := loadProviderMetadataFromURL( client, dnsURL, already, logging) lg(dnsResult, dnsURL) return dnsResult From 1854678409cd77a4c14946ab008d64de8d8a92c5 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 24 Mar 2023 18:59:38 +0100 Subject: [PATCH 02/14] Started with cleaning up the provider metadata loading --- csaf/util.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/csaf/util.go b/csaf/util.go index 1e7cf97..ecef653 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -22,23 +22,6 @@ import ( "github.com/csaf-poc/csaf_distribution/util" ) -// LoadedProviderMetadata represents a loaded provider metadata. -type LoadedProviderMetadata struct { - // URL is location where the document was found. - URL string - // Document is the de-serialized JSON document. - Document any - // Hash is a SHA256 sum over the document. - Hash []byte - // Messages are the error message happened while loading. - Messages []string -} - -// Valid returns true if the loaded document is valid. -func (lpm *LoadedProviderMetadata) Valid() bool { - return lpm != nil && lpm.Document != nil && lpm.Hash != nil -} - // defaultLogging generates a logging function if given is nil. func defaultLogging( logging func(format string, args ...any), From 21477e8004424bc23454f0d8516464de4a536f8f Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 18 Apr 2023 14:21:56 +0200 Subject: [PATCH 03/14] Add missing file --- csaf/providermetaloader.go | 82 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 csaf/providermetaloader.go diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go new file mode 100644 index 0000000..490c9a7 --- /dev/null +++ b/csaf/providermetaloader.go @@ -0,0 +1,82 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +package csaf + +import ( + "errors" + "log" + "strings" + + "github.com/csaf-poc/csaf_distribution/util" +) + +// ProviderMetadataLoader helps load provider-metadata.json from +// the various locations. +type ProviderMetadataLoader struct { + client *util.Client + logging func(string, ...any) +} + +// LoadedProviderMetadata represents a loaded provider metadata. +type LoadedProviderMetadata struct { + // URL is location where the document was found. + URL string + // Document is the de-serialized JSON document. + Document any + // Hash is a SHA256 sum over the document. + Hash []byte + // Messages are the error message happened while loading. + Messages []string +} + +// Valid returns true if the loaded document is valid. +func (lpm *LoadedProviderMetadata) Valid() bool { + return lpm != nil && lpm.Document != nil && lpm.Hash != nil +} + +// NewProviderMetadataLoader create a new loader. +func NewProviderMetadataLoader( + client *util.Client, + logging func(string, ...any), +) *ProviderMetadataLoader { + + // If no logging was given log to stdout. + if logging == nil { + logging = func(format string, args ...any) { + log.Printf("ProviderMetadataLoader: "+format+"\n", args...) + } + } + return &ProviderMetadataLoader{ + client: client, + logging: logging, + } +} + +// Load loads a provider metadata for a given path. +// If the domain starts with `https://` it only attemps to load +// the data from that URL. +func (pmdl *ProviderMetadataLoader) Load(path string) (*LoadedProviderMetadata, error) { + + // check direct path + if strings.HasPrefix(path, "https://") { + return pmdl.loadFromURL(path) + } + + // TODO: Implement me! + return nil, errors.New("not implemented, yet") +} + +// loadFromURL loads a provider metadata from a given URL. +func (pmdl *ProviderMetadataLoader) loadFromURL(path string) (*LoadedProviderMetadata, error) { + + _ = path + + // TODO: Implement me! + return nil, errors.New("not implemented, yet") +} From dd15eea48eb354c6e1858072c932251878644ed4 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Fri, 12 May 2023 11:05:05 +0200 Subject: [PATCH 04/14] Fill typed messages for pmd loading --- csaf/providermetaloader.go | 21 ++++++++++++++++++++- csaf/util.go | 24 +++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 490c9a7..4924aa0 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -23,6 +23,25 @@ type ProviderMetadataLoader struct { logging func(string, ...any) } +// ProviderMetadataLoadMessageType is the type of the message. +type ProviderMetadataLoadMessageType int + +const ( + //JSONDecodingFailed indicates problems with JSON decoding + JSONDecodingFailed ProviderMetadataLoadMessageType = iota + // SchemaValidationFailed indicates a general problem with schema validation. + SchemaValidationFailed + // SchemaValidationFailedDetail is a failure detail in schema validation. + SchemaValidationFailedDetail +) + +// ProviderMetadataLoadMessage is a message generated while loading +// a provider meta data file. +type ProviderMetadataLoadMessage struct { + Type ProviderMetadataLoadMessageType + Message string +} + // LoadedProviderMetadata represents a loaded provider metadata. type LoadedProviderMetadata struct { // URL is location where the document was found. @@ -32,7 +51,7 @@ type LoadedProviderMetadata struct { // Hash is a SHA256 sum over the document. Hash []byte // Messages are the error message happened while loading. - Messages []string + Messages []ProviderMetadataLoadMessage } // Valid returns true if the loaded document is valid. diff --git a/csaf/util.go b/csaf/util.go index ecef653..0abe7c2 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -93,21 +93,31 @@ func loadProviderMetadataFromURL( // We have loaded it the first time. if err != nil { - result.Messages = []string{fmt.Sprintf("%s: Decoding JSON failed: %v", url, err)} + result.Messages = []ProviderMetadataLoadMessage{{ + Type: JSONDecodingFailed, + Message: fmt.Sprintf("%s: Decoding JSON failed: %v", url, err), + }} storeLoaded() return &result } switch errors, err := ValidateProviderMetadata(doc); { case err != nil: - result.Messages = []string{ - fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)} + result.Messages = []ProviderMetadataLoadMessage{{ + Type: SchemaValidationFailed, + Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err), + }} case len(errors) > 0: - result.Messages = []string{ - fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err)} + result.Messages = []ProviderMetadataLoadMessage{{ + Type: SchemaValidationFailed, + Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err), + }} for _, msg := range errors { - result.Messages = append(result.Messages, strings.ReplaceAll(msg, `%`, `%%`)) + result.Messages = append(result.Messages, ProviderMetadataLoadMessage{ + Type: SchemaValidationFailedDetail, + Message: strings.ReplaceAll(msg, `%`, `%%`), + }) } default: // Only store in result if validation passed. @@ -193,7 +203,7 @@ func LoadProviderMetadataForDomain( } alreadyLogged[result] = url for _, msg := range result.Messages { - logging(msg) + logging(msg.Message) } } From e0928f58add2c81e3fe27eaf9b8a402a385fb0d5 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 00:35:35 +0200 Subject: [PATCH 05/14] Port over logic to new PMD loader --- csaf/providermetaloader.go | 291 +++++++++++++++++++++++++++++++++---- 1 file changed, 265 insertions(+), 26 deletions(-) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 4924aa0..1c321ed 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -9,8 +9,12 @@ package csaf import ( - "errors" - "log" + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" "strings" "github.com/csaf-poc/csaf_distribution/util" @@ -19,8 +23,9 @@ import ( // ProviderMetadataLoader helps load provider-metadata.json from // the various locations. type ProviderMetadataLoader struct { - client *util.Client - logging func(string, ...any) + client util.Client + already map[string]*LoadedProviderMetadata + messages ProviderMetadataLoadMessages } // ProviderMetadataLoadMessageType is the type of the message. @@ -33,6 +38,15 @@ const ( SchemaValidationFailed // SchemaValidationFailedDetail is a failure detail in schema validation. SchemaValidationFailedDetail + // HTTPFailed indicates that loading on HTTP level failed. + HTTPFailed + // ExtraProviderMetadataFound indicates an extra PMD found in security.txt. + ExtraProviderMetadataFound + // WellknownSecurityMismatch indicates that the PMDs found under wellknown and + // in the security do not match. + WellknownSecurityMismatch + // IgnoreProviderMetadata indicates that a extra PMD was ignored. + IgnoreProviderMetadata ) // ProviderMetadataLoadMessage is a message generated while loading @@ -42,6 +56,9 @@ type ProviderMetadataLoadMessage struct { Message string } +// ProviderMetadataLoadMessages is a list of loading messages. +type ProviderMetadataLoadMessages []ProviderMetadataLoadMessage + // LoadedProviderMetadata represents a loaded provider metadata. type LoadedProviderMetadata struct { // URL is location where the document was found. @@ -51,7 +68,31 @@ type LoadedProviderMetadata struct { // Hash is a SHA256 sum over the document. Hash []byte // Messages are the error message happened while loading. - Messages []ProviderMetadataLoadMessage + Messages ProviderMetadataLoadMessages +} + +// Add appends a message to the list of loading messages. +func (pmlm *ProviderMetadataLoadMessages) Add( + typ ProviderMetadataLoadMessageType, + msg string, +) { + *pmlm = append(*pmlm, ProviderMetadataLoadMessage{ + Type: typ, + Message: msg, + }) +} + +// AppendUnique appends unique messages from a second list. +func (pmlm *ProviderMetadataLoadMessages) AppendUnique(other ProviderMetadataLoadMessages) { +next: + for _, o := range other { + for _, m := range *pmlm { + if m == o { + continue next + } + } + *pmlm = append(*pmlm, o) + } } // Valid returns true if the loaded document is valid. @@ -60,42 +101,240 @@ func (lpm *LoadedProviderMetadata) Valid() bool { } // NewProviderMetadataLoader create a new loader. -func NewProviderMetadataLoader( - client *util.Client, - logging func(string, ...any), -) *ProviderMetadataLoader { - - // If no logging was given log to stdout. - if logging == nil { - logging = func(format string, args ...any) { - log.Printf("ProviderMetadataLoader: "+format+"\n", args...) - } - } +func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader { return &ProviderMetadataLoader{ client: client, - logging: logging, + already: map[string]*LoadedProviderMetadata{}, } } // Load loads a provider metadata for a given path. // If the domain starts with `https://` it only attemps to load // the data from that URL. -func (pmdl *ProviderMetadataLoader) Load(path string) (*LoadedProviderMetadata, error) { +func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata { - // check direct path - if strings.HasPrefix(path, "https://") { - return pmdl.loadFromURL(path) + // Check direct path + if strings.HasPrefix(domain, "https://") { + lpmd, err := pmdl.loadFromURL(domain) + if err != nil { + lpmd = new(LoadedProviderMetadata) + lpmd.Messages.Add(HTTPFailed, err.Error()) + } + return lpmd } - // TODO: Implement me! - return nil, errors.New("not implemented, yet") + // First try the well-known path. + wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" + + wellknownResult, err := pmdl.loadFromURL(wellknownURL) + if err != nil { + pmdl.messages.Add(HTTPFailed, err.Error()) + } + + // Valid provider metadata under well-known. + var wellknownGood *LoadedProviderMetadata + + // We have a candidate. + if wellknownResult.Valid() { + wellknownGood = wellknownResult + } + + // Next load the PMDs from security.txt + secURL := "https://" + domain + "/.well-known/security.txt" + secResults := pmdl.loadFromSecurity(secURL) + + // Filter out the results which are valid. + var secGoods []*LoadedProviderMetadata + + for _, result := range secResults { + if len(result.Messages) > 0 { + // If there where validation issues append them + // to the overall report + pmdl.messages.AppendUnique(pmdl.messages) + } else { + secGoods = append(secGoods, result) + } + } + + // Mention extra CSAF entries in security.txt. + ignoreExtras := func() { + for _, extra := range secGoods[1:] { + pmdl.messages.Add( + ExtraProviderMetadataFound, + fmt.Sprintf("Ignoring extra CSAF entry in security.txt: %s", extra.URL)) + } + } + + // security.txt contains good entries. + if len(secGoods) > 0 { + // we already have a good wellknown, take it. + if wellknownGood != nil { + // check if first of security urls is identical to wellknown. + if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) { + ignoreExtras() + } else { + // Complaint about not matching. + pmdl.messages.Add( + WellknownSecurityMismatch, + "First entry of security.txt and well-known don't match.") + // List all the security urls. + for _, sec := range secGoods { + pmdl.messages.Add( + IgnoreProviderMetadata, + fmt.Sprintf("Ignoring CSAF entry in security.txt: %s", sec.URL)) + } + } + // Take the good well-known. + wellknownGood.Messages.AppendUnique(pmdl.messages) + return wellknownGood + } + + // Don't have well-known. Take first good from security.txt. + ignoreExtras() + secGoods[0].Messages.AppendUnique(pmdl.messages) + return secGoods[0] + } + + // If we have a good well-known take it. + if wellknownGood != nil { + wellknownGood.Messages.AppendUnique(pmdl.messages) + return wellknownGood + } + + // Last resort: fall back to DNS. + dnsURL := "https://csaf.data.security." + domain + + dnsResult, err := pmdl.loadFromURL(dnsURL) + if err != nil { + dnsResult = new(LoadedProviderMetadata) + pmdl.messages.Add( + HTTPFailed, + err.Error()) + } + dnsResult.Messages.AppendUnique(pmdl.messages) + return dnsResult +} + +// loadFromSecurity loads the PMDs mentioned in the security.txt. +func (pmdl *ProviderMetadataLoader) loadFromSecurity(path string) []*LoadedProviderMetadata { + + res, err := pmdl.client.Get(path) + if err != nil { + pmdl.messages.Add( + HTTPFailed, + fmt.Sprintf("Fetching %q failed: %v", path, err)) + return nil + } + if res.StatusCode != http.StatusOK { + pmdl.messages.Add( + HTTPFailed, + fmt.Sprintf("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode)) + return nil + } + + // Extract all potential URLs from CSAF. + urls, err := func() ([]string, error) { + defer res.Body.Close() + return ExtractProviderURL(res.Body, true) + }() + + if err != nil { + pmdl.messages.Add( + HTTPFailed, + fmt.Sprintf("Loading %q failed: %v", path, err)) + return nil + } + + var loaded []*LoadedProviderMetadata + + // Load the URLs +nextURL: + for _, url := range urls { + lpmd, err := pmdl.loadFromURL(url) + // If loading failed note it down. + if err != nil { + pmdl.messages.Add( + HTTPFailed, + fmt.Sprintf("Loading %q failed: %v", url, err)) + continue + } + // Check for duplicates + for _, l := range loaded { + if l == lpmd { + continue nextURL + } + } + loaded = append(loaded, lpmd) + } + + return loaded } // loadFromURL loads a provider metadata from a given URL. func (pmdl *ProviderMetadataLoader) loadFromURL(path string) (*LoadedProviderMetadata, error) { - _ = path + res, err := pmdl.client.Get(path) + if err != nil { + return nil, fmt.Errorf("fetching %q failed: %v", path, err) + } + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetching %q failed: %s (%d)", path, res.Status, res.StatusCode) + } - // TODO: Implement me! - return nil, errors.New("not implemented, yet") + // TODO: Check for application/json and log it. + + defer res.Body.Close() + + // Calculate checksum for later comparison. + hash := sha256.New() + + result := LoadedProviderMetadata{URL: path} + + tee := io.TeeReader(res.Body, hash) + + var doc any + + if err := json.NewDecoder(tee).Decode(&doc); err != nil { + return nil, fmt.Errorf("JSON decoding failed: %w", err) + } + + // Before checking the err lets check if we had the same + // document before. If so it will have failed parsing before. + + sum := hash.Sum(nil) + key := string(sum) + + // If we already have loaded it return the cached result. + if r := pmdl.already[key]; r != nil { + return r, nil + } + + // write it back as loaded + + switch errors, err := ValidateProviderMetadata(doc); { + case err != nil: + result.Messages = []ProviderMetadataLoadMessage{{ + Type: SchemaValidationFailed, + Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err), + }} + + case len(errors) > 0: + result.Messages = []ProviderMetadataLoadMessage{{ + Type: SchemaValidationFailed, + Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err), + }} + for _, msg := range errors { + result.Messages = append(result.Messages, ProviderMetadataLoadMessage{ + Type: SchemaValidationFailedDetail, + Message: strings.ReplaceAll(msg, `%`, `%%`), + }) + } + default: + // Only store in result if validation passed. + result.Document = doc + result.Hash = sum + } + + pmdl.already[key] = &result + return &result, nil } From 2e968b197d0ad9accf8c2b8b90a321125e64d01c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 08:47:18 +0200 Subject: [PATCH 06/14] Removed old pmd loader. --- cmd/csaf_aggregator/processor.go | 14 +- cmd/csaf_checker/processor.go | 6 +- cmd/csaf_downloader/downloader.go | 15 +- csaf/util.go | 282 ------------------------------ 4 files changed, 25 insertions(+), 292 deletions(-) diff --git a/cmd/csaf_aggregator/processor.go b/cmd/csaf_aggregator/processor.go index b665a0b..7ba511b 100644 --- a/cmd/csaf_aggregator/processor.go +++ b/cmd/csaf_aggregator/processor.go @@ -79,11 +79,17 @@ func (w *worker) createDir() (string, error) { func (w *worker) locateProviderMetadata(domain string) error { - lpmd := csaf.LoadProviderMetadataForDomain( - w.client, domain, func(format string, args ...any) { + loader := csaf.NewProviderMetadataLoader(w.client) + + lpmd := loader.Load(domain) + + if w.processor.cfg.Verbose { + for i := range lpmd.Messages { log.Printf( - "Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...) - }) + "Loading provider-metadata.json of %q: %s\n", + domain, lpmd.Messages[i].Message) + } + } if !lpmd.Valid() { return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 4124455..3d98f3c 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -1114,7 +1114,11 @@ func (p *processor) checkProviderMetadata(domain string) error { client := p.httpClient() - lpmd := csaf.LoadProviderMetadataForDomain(client, domain, p.badProviderMetadata.warn) + loader := csaf.NewProviderMetadataLoader(client) + + lpmd := loader.Load(domain) + + // TODO: Sort results into reports. if !lpmd.Valid() { p.badProviderMetadata.error("No valid provider-metadata.json found.") diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 36091f5..4457b09 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -118,11 +118,16 @@ func (d *downloader) httpClient() util.Client { func (d *downloader) download(ctx context.Context, domain string) error { client := d.httpClient() - lpmd := csaf.LoadProviderMetadataForDomain( - client, domain, func(format string, args ...any) { - log.Printf( - "Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...) - }) + loader := csaf.NewProviderMetadataLoader(client) + + lpmd := loader.Load(domain) + + if d.opts.Verbose { + for i := range lpmd.Messages { + log.Printf("Loading provider-metadata.json for %q: %s\n", + domain, lpmd.Messages[i].Message) + } + } if !lpmd.Valid() { return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) diff --git a/csaf/util.go b/csaf/util.go index 0abe7c2..f192f09 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -10,292 +10,10 @@ package csaf import ( "bufio" - "bytes" - "crypto/sha256" - "encoding/json" - "fmt" "io" - "log" - "net/http" "strings" - - "github.com/csaf-poc/csaf_distribution/util" ) -// defaultLogging generates a logging function if given is nil. -func defaultLogging( - logging func(format string, args ...any), - prefix, suffix string, -) func(format string, args ...any) { - - if logging != nil { - return logging - } - return func(format string, args ...any) { - log.Printf(prefix+format+suffix, args...) - } -} - -// loadProviderMetadataFromURL loads a provider metadata from a given URL. -// Returns nil if the document was not found. -func loadProviderMetadataFromURL( - client util.Client, - url string, - already map[string]*LoadedProviderMetadata, - logging func(format string, args ...any), -) *LoadedProviderMetadata { - - logging = defaultLogging(logging, "loadProviderMetadataFromURL: ", "\n") - - res, err := client.Get(url) - if err != nil { - logging("Fetching %q failed: %v", url, err) - return nil - } - if res.StatusCode != http.StatusOK { - logging("Fetching %q failed: %s (%d)", url, res.Status, res.StatusCode) - return nil - } - - // TODO: Check for application/json and log it. - - defer res.Body.Close() - - // Calculate checksum for later comparison. - hash := sha256.New() - - result := LoadedProviderMetadata{URL: url} - - tee := io.TeeReader(res.Body, hash) - - var doc any - - err = json.NewDecoder(tee).Decode(&doc) - // Before checking the err lets check if we had the same - // document before. If so it will have failed parsing before. - - sum := hash.Sum(nil) - - var key string - if already != nil { - key = string(sum) - if r, ok := already[key]; ok { - return r - } - } - - // write it back as loaded - storeLoaded := func() { - if already != nil { - already[key] = &result - } - } - - // We have loaded it the first time. - if err != nil { - result.Messages = []ProviderMetadataLoadMessage{{ - Type: JSONDecodingFailed, - Message: fmt.Sprintf("%s: Decoding JSON failed: %v", url, err), - }} - storeLoaded() - return &result - } - - switch errors, err := ValidateProviderMetadata(doc); { - case err != nil: - result.Messages = []ProviderMetadataLoadMessage{{ - Type: SchemaValidationFailed, - Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err), - }} - - case len(errors) > 0: - result.Messages = []ProviderMetadataLoadMessage{{ - Type: SchemaValidationFailed, - Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", url, err), - }} - for _, msg := range errors { - result.Messages = append(result.Messages, ProviderMetadataLoadMessage{ - Type: SchemaValidationFailedDetail, - Message: strings.ReplaceAll(msg, `%`, `%%`), - }) - } - default: - // Only store in result if validation passed. - result.Document = doc - result.Hash = sum - } - - storeLoaded() - return &result -} - -// loadProviderMetadatasFromSecurity loads a secturity.txt, -// extracts and the CSAF urls from the document. -// Returns nil if no url was successfully found. -func loadProviderMetadatasFromSecurity( - client util.Client, - path string, - already map[string]*LoadedProviderMetadata, - logging func(format string, args ...any), -) []*LoadedProviderMetadata { - - logging = defaultLogging(logging, "loadProviderMetadataFromSecurity: ", "\n") - - res, err := client.Get(path) - if err != nil { - logging("Fetching %q failed: %v", path, err) - return nil - } - if res.StatusCode != http.StatusOK { - logging("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode) - return nil - } - - // Extract all potential URLs from CSAF. - urls, err := func() ([]string, error) { - defer res.Body.Close() - return ExtractProviderURL(res.Body, true) - }() - - if err != nil { - // Treat as not found - return nil - } - - var results []*LoadedProviderMetadata - - // Load the URLs - for _, url := range urls { - if result := loadProviderMetadataFromURL( - client, url, already, logging, - ); result.Valid() { - results = append(results, result) - } - } - - return results -} - -// LoadProviderMetadataForDomain loads a provider metadata for a given domain. -// Returns nil if no provider metadata (PMD) was found. -// If the domain starts with `https://` it only attemps to load -// the data from that URL. -// The logging can be used to track the errors happening while loading. -func LoadProviderMetadataForDomain( - client util.Client, - domain string, - logging func(format string, args ...any), -) *LoadedProviderMetadata { - - logging = defaultLogging(logging, "LoadProviderMetadataForDomain: ", "\n") - - // As many URLs may lead to the same content only log once per content. - alreadyLogged := map[*LoadedProviderMetadata]string{} - - lg := func(result *LoadedProviderMetadata, url string) { - if result == nil { - logging("%q not found.", url) - return - } - if other := alreadyLogged[result]; other != "" { - logging("%q is same %q.", url, other) - return - } - alreadyLogged[result] = url - for _, msg := range result.Messages { - logging(msg.Message) - } - } - - // keey track of already loaded pmds. - already := map[string]*LoadedProviderMetadata{} - - // check direct path - if strings.HasPrefix(domain, "https://") { - result := loadProviderMetadataFromURL( - client, domain, already, logging) - lg(result, domain) - return result - } - - // Valid provider metadata under well-known. - var wellknownGood *LoadedProviderMetadata - - // First try the well-known path. - wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" - wellknownResult := loadProviderMetadataFromURL( - client, wellknownURL, already, logging) - lg(wellknownResult, wellknownURL) - - // We have a candidate. - if wellknownResult.Valid() { - wellknownGood = wellknownResult - } - - // Next load the PMDs from security.txt - secURL := "https://" + domain + "/.well-known/security.txt" - secResults := loadProviderMetadatasFromSecurity( - client, secURL, already, logging) - - if len(secResults) == 0 { - logging("%s failed to load.", secURL) - } else { - // Filter out the results which are valid. - var secGoods []*LoadedProviderMetadata - - for _, result := range secResults { - if len(result.Messages) > 0 { - lg(result, result.URL) - } else { - secGoods = append(secGoods, result) - } - } - - // security.txt contains good entries. - if len(secGoods) > 0 { - // we already have a good wellknown, take it. - if wellknownGood != nil { - // check if first of security urls is identical to wellknown. - if bytes.Equal(wellknownGood.Hash, secGoods[0].Hash) { - // Mention extra CSAF entries - for _, extra := range secGoods[1:] { - logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL) - } - } else { - // Complaint about not matching. - logging("First entry of security.txt and well-known don't match.") - // List all the security urls. - for _, sec := range secGoods { - logging("Ignoring CSAF entry in security.txt: %s", sec.URL) - } - } - // Take the good well-known. - return wellknownGood - } - - // Don't have well-known. Take first good from security.txt. - // Mention extra CSAF entries - for _, extra := range secGoods[1:] { - logging("Ignoring extra CSAF entry in security.txt: %s", extra.URL) - } - - return secGoods[0] - } - } - - // If we have a good well-known take it. - if wellknownGood != nil { - return wellknownGood - } - - // Last resort: fall back to DNS. - dnsURL := "https://csaf.data.security." + domain - dnsResult := loadProviderMetadataFromURL( - client, dnsURL, already, logging) - lg(dnsResult, dnsURL) - return dnsResult -} - // ExtractProviderURL extracts URLs of provider metadata. // If all is true all URLs are returned. Otherwise only the first is returned. func ExtractProviderURL(r io.Reader, all bool) ([]string, error) { From 018a1814f0e8fe340f53c0b7f47fc12d67718fa0 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 10:43:16 +0200 Subject: [PATCH 07/14] Fixed problem with JSON decoding in loading PMD --- csaf/providermetaloader.go | 70 ++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 1c321ed..7e333e6 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -115,21 +115,13 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata // Check direct path if strings.HasPrefix(domain, "https://") { - lpmd, err := pmdl.loadFromURL(domain) - if err != nil { - lpmd = new(LoadedProviderMetadata) - lpmd.Messages.Add(HTTPFailed, err.Error()) - } - return lpmd + return pmdl.loadFromURL(domain) } // First try the well-known path. wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" - wellknownResult, err := pmdl.loadFromURL(wellknownURL) - if err != nil { - pmdl.messages.Add(HTTPFailed, err.Error()) - } + wellknownResult := pmdl.loadFromURL(wellknownURL) // Valid provider metadata under well-known. var wellknownGood *LoadedProviderMetadata @@ -203,16 +195,7 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata // Last resort: fall back to DNS. dnsURL := "https://csaf.data.security." + domain - - dnsResult, err := pmdl.loadFromURL(dnsURL) - if err != nil { - dnsResult = new(LoadedProviderMetadata) - pmdl.messages.Add( - HTTPFailed, - err.Error()) - } - dnsResult.Messages.AppendUnique(pmdl.messages) - return dnsResult + return pmdl.loadFromURL(dnsURL) } // loadFromSecurity loads the PMDs mentioned in the security.txt. @@ -250,12 +233,10 @@ func (pmdl *ProviderMetadataLoader) loadFromSecurity(path string) []*LoadedProvi // Load the URLs nextURL: for _, url := range urls { - lpmd, err := pmdl.loadFromURL(url) + lpmd := pmdl.loadFromURL(url) // If loading failed note it down. - if err != nil { - pmdl.messages.Add( - HTTPFailed, - fmt.Sprintf("Loading %q failed: %v", url, err)) + if !lpmd.Valid() { + pmdl.messages.AppendUnique(lpmd.Messages) continue } // Check for duplicates @@ -271,14 +252,22 @@ nextURL: } // loadFromURL loads a provider metadata from a given URL. -func (pmdl *ProviderMetadataLoader) loadFromURL(path string) (*LoadedProviderMetadata, error) { +func (pmdl *ProviderMetadataLoader) loadFromURL(path string) *LoadedProviderMetadata { + + result := LoadedProviderMetadata{URL: path} res, err := pmdl.client.Get(path) if err != nil { - return nil, fmt.Errorf("fetching %q failed: %v", path, err) + result.Messages.Add( + HTTPFailed, + fmt.Sprintf("fetching %q failed: %v", path, err)) + return &result } if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("fetching %q failed: %s (%d)", path, res.Status, res.StatusCode) + result.Messages.Add( + HTTPFailed, + fmt.Sprintf("fetching %q failed: %s (%d)", path, res.Status, res.StatusCode)) + return &result } // TODO: Check for application/json and log it. @@ -288,14 +277,15 @@ func (pmdl *ProviderMetadataLoader) loadFromURL(path string) (*LoadedProviderMet // Calculate checksum for later comparison. hash := sha256.New() - result := LoadedProviderMetadata{URL: path} - tee := io.TeeReader(res.Body, hash) var doc any if err := json.NewDecoder(tee).Decode(&doc); err != nil { - return nil, fmt.Errorf("JSON decoding failed: %w", err) + result.Messages.Add( + JSONDecodingFailed, + fmt.Sprintf("JSON decoding failed: %v", err)) + return &result } // Before checking the err lets check if we had the same @@ -306,17 +296,16 @@ func (pmdl *ProviderMetadataLoader) loadFromURL(path string) (*LoadedProviderMet // If we already have loaded it return the cached result. if r := pmdl.already[key]; r != nil { - return r, nil + return r } // write it back as loaded switch errors, err := ValidateProviderMetadata(doc); { case err != nil: - result.Messages = []ProviderMetadataLoadMessage{{ - Type: SchemaValidationFailed, - Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err), - }} + result.Messages.Add( + SchemaValidationFailed, + fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err)) case len(errors) > 0: result.Messages = []ProviderMetadataLoadMessage{{ @@ -324,10 +313,9 @@ func (pmdl *ProviderMetadataLoader) loadFromURL(path string) (*LoadedProviderMet Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err), }} for _, msg := range errors { - result.Messages = append(result.Messages, ProviderMetadataLoadMessage{ - Type: SchemaValidationFailedDetail, - Message: strings.ReplaceAll(msg, `%`, `%%`), - }) + result.Messages.Add( + SchemaValidationFailedDetail, + strings.ReplaceAll(msg, `%`, `%%`)) } default: // Only store in result if validation passed. @@ -336,5 +324,5 @@ func (pmdl *ProviderMetadataLoader) loadFromURL(path string) (*LoadedProviderMet } pmdl.already[key] = &result - return &result, nil + return &result } From bd7831d7c32273c3ec77c5d49ab9d438013ade94 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 12:12:42 +0200 Subject: [PATCH 08/14] Build reporters from role --- cmd/csaf_checker/main.go | 81 +++++++++++++++++++++++++++-------- cmd/csaf_checker/processor.go | 29 +++++++------ 2 files changed, 79 insertions(+), 31 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 645a17a..c1bcd03 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -21,7 +21,9 @@ import ( "log" "net/http" "os" + "sort" + "github.com/csaf-poc/csaf_distribution/csaf" "github.com/csaf-poc/csaf_distribution/util" "github.com/jessevdk/go-flags" ) @@ -140,26 +142,69 @@ func writeReport(report *Report, opts *options) error { return writer(report, w) } +var reporters = [23]reporter{ + &validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}}, + &filenameReporter{baseReporter{num: 2, description: "Filename"}}, + &tlsReporter{baseReporter{num: 3, description: "TLS"}}, + nil, // TODO: Add 4: TLP:WHITE + nil, // TODO: Add 5: TLP:AMBER and TLP:RED + &redirectsReporter{baseReporter{num: 6, description: "Redirects"}}, + &providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}}, + &securityReporter{baseReporter{num: 8, description: "security.txt"}}, + &wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}}, + &dnsPathReporter{baseReporter{num: 10, description: "DNS path"}}, + &oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}}, + &indexReporter{baseReporter{num: 12, description: "index.txt"}}, + &changesReporter{baseReporter{num: 13, description: "changes.csv"}}, + &directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}}, + nil, // TODO: Add 15: ROLIE feed + nil, // TODO: Add 16: ROLIE service document + nil, // TODO: Add 17: ROLIE category document + &integrityReporter{baseReporter{num: 18, description: "Integrity"}}, + &signaturesReporter{baseReporter{num: 19, description: "Signatures"}}, + &publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}}, + nil, // TODO: Add 21: List of CSAF providers + nil, // TODO: Add 22: Two disjoint issuing parties + nil, // TODO: Add 23: Mirror +} + +var roleImplies = map[csaf.MetadataRole][]csaf.MetadataRole{ + csaf.MetadataRoleProvider: {csaf.MetadataRolePublisher}, + csaf.MetadataRoleTrustedProvider: {csaf.MetadataRoleProvider}, +} + +func requirements(role csaf.MetadataRole) [][2]int { + var own [][2]int + switch role { + case csaf.MetadataRoleTrustedProvider: + own = [][2]int{{18, 20}} + case csaf.MetadataRoleProvider: + own = [][2]int{{5, 7}, {8, 10}, {11, 14}, {15, 17}} + case csaf.MetadataRolePublisher: + own = [][2]int{{1, 4}} + } + for _, base := range roleImplies[role] { + own = append(own, requirements(base)...) + } + return own +} + // buildReporters initializes each report by assigning a number and description to it. // It returns an array of the reporter interface type. -func buildReporters() []reporter { - return []reporter{ - &validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}}, - &filenameReporter{baseReporter{num: 2, description: "Filename"}}, - &tlsReporter{baseReporter{num: 3, description: "TLS"}}, - &redirectsReporter{baseReporter{num: 6, description: "Redirects"}}, - &providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}}, - &securityReporter{baseReporter{num: 8, description: "security.txt"}}, - &wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}}, - &dnsPathReporter{baseReporter{num: 10, description: "DNS path"}}, - &oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}}, - &indexReporter{baseReporter{num: 12, description: "index.txt"}}, - &changesReporter{baseReporter{num: 13, description: "changes.csv"}}, - &directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}}, - &integrityReporter{baseReporter{num: 18, description: "Integrity"}}, - &signaturesReporter{baseReporter{num: 19, description: "Signatures"}}, - &publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}}, +func buildReporters(role csaf.MetadataRole) []reporter { + var reps []reporter + reqs := requirements(role) + // sort to have them ordered by there number. + sort.Slice(reqs, func(i, j int) bool { return reqs[i][0] < reqs[j][0] }) + for _, req := range reqs { + from, to := req[0]-1, req[1]-1 + for i := from; i <= to; i++ { + if rep := reporters[i]; rep != nil { + reps = append(reps, rep) + } + } } + return reps } // run uses a processor to check all the given domains or direct urls @@ -170,7 +215,7 @@ func run(opts *options, domains []string) (*Report, error) { return nil, err } defer p.close() - return p.run(buildReporters(), domains) + return p.run(domains) } func main() { diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 3d98f3c..c928814 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -223,7 +223,7 @@ func (p *processor) clean() { // run calls checkDomain function for each domain in the given "domains" parameter. // Then it calls the report method on each report from the given "reporters" parameter for each domain. // It returns a pointer to the report and nil, otherwise an error. -func (p *processor) run(reporters []reporter, domains []string) (*Report, error) { +func (p *processor) run(domains []string) (*Report, error) { report := Report{ Date: ReportTime{Time: time.Now().UTC()}, @@ -231,19 +231,24 @@ func (p *processor) run(reporters []reporter, domains []string) (*Report, error) } for _, d := range domains { - if err := p.checkDomain(d); err != nil { - if err == errContinue || err == errStop { - continue + if p.checkProviderMetadata(d) { + if err := p.checkDomain(d); err != nil { + if err == errContinue || err == errStop { + continue + } + return nil, err } - return nil, err } domain := &Domain{Name: d} - for _, r := range reporters { - r.report(p, domain) - } if err := p.fillMeta(domain); err != nil { log.Printf("Filling meta data failed: %v\n", err) + // reporters depend on role. + continue + } + + for _, r := range buildReporters(*domain.Role) { + r.report(p, domain) } report.Domains = append(report.Domains, domain) @@ -287,7 +292,6 @@ func (p *processor) domainChecks(domain string) []func(*processor, string) error direct := strings.HasPrefix(domain, "https://") checks := []func(*processor, string) error{ - (*processor).checkProviderMetadata, (*processor).checkPGPKeys, } @@ -1107,8 +1111,7 @@ func (p *processor) checkListing(string) error { // decodes, and validates against the JSON schema. // According to the result, the respective error messages added to // badProviderMetadata. -// It returns nil if all checks are passed. -func (p *processor) checkProviderMetadata(domain string) error { +func (p *processor) checkProviderMetadata(domain string) bool { p.badProviderMetadata.use() @@ -1123,14 +1126,14 @@ func (p *processor) checkProviderMetadata(domain string) error { if !lpmd.Valid() { p.badProviderMetadata.error("No valid provider-metadata.json found.") p.badProviderMetadata.error("STOPPING here - cannot perform other checks.") - return errStop + return false } p.pmdURL = lpmd.URL p.pmd256 = lpmd.Hash p.pmd = lpmd.Document - return nil + return true } // checkSecurity checks the security.txt file by making HTTP request to fetch it. From aeff51189564402b94a6c2ed8c4374cf449a650c Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer Date: Mon, 15 May 2023 13:49:27 +0200 Subject: [PATCH 09/14] Add reporters for missing requirements and their respective report functions --- cmd/csaf_checker/reporters.go | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/cmd/csaf_checker/reporters.go b/cmd/csaf_checker/reporters.go index 19772a8..1d3100f 100644 --- a/cmd/csaf_checker/reporters.go +++ b/cmd/csaf_checker/reporters.go @@ -22,6 +22,8 @@ type ( validReporter struct{ baseReporter } filenameReporter struct{ baseReporter } tlsReporter struct{ baseReporter } + tlpWhiteReporter struct{ baseReporter } + tlpAmberRedReporter struct{ baseReporter } redirectsReporter struct{ baseReporter } providerMetadataReport struct{ baseReporter } securityReporter struct{ baseReporter } @@ -31,9 +33,15 @@ type ( indexReporter struct{ baseReporter } changesReporter struct{ baseReporter } directoryListingsReporter struct{ baseReporter } + rolieFeedReporter struct{ baseReporter } + rolieServiceReporter struct{ baseReporter } + rolieCategoryReporter struct{ baseReporter } integrityReporter struct{ baseReporter } signaturesReporter struct{ baseReporter } publicPGPKeyReporter struct{ baseReporter } + listReporter struct{ baseReporter } + hasTwoReporter struct{ baseReporter } + mirrorReporter struct{ baseReporter } ) func (bc *baseReporter) requirement(domain *Domain) *Requirement { @@ -115,6 +123,21 @@ func (r *tlsReporter) report(p *processor, domain *Domain) { req.message(ErrorType, urls...) } +// report tests if a document labeled TLP:WHITE +// is freely accessible and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *tlpWhiteReporter) report(p *processor, domain *Domain) { + // TODO +} + +// report tests if a document labeled TLP:AMBER +// or TLP:RED is access protected +// and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *tlpAmberRedReporter) report(p *processor, domain *Domain) { + // TODO +} + // report tests if redirects are used and sets the "message" field value // of the "Requirement" struct as a result of that. func (r *redirectsReporter) report(p *processor, domain *Domain) { @@ -269,6 +292,31 @@ func (r *directoryListingsReporter) report(p *processor, domain *Domain) { req.Messages = p.badDirListings } +// report checks whether there is only a single ROLIE feed for a +// given TLP level and whether any of the TLP levels +// TLP:WHITE, TLP:GREEN or unlabeled exists and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *rolieFeedReporter) report(p *processor, domain *Domain) { + // TODO +} + +// report tests whether a ROLIE service document is used and if so, +// whether it is a [RFC8322] conform JSON file that lists the +// ROLIE feed documents and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *rolieServiceReporter) report(p *processor, domain *Domain) { + // TODO +} + +// report tests whether a ROLIE category document is used and if so, +// whether it is a [RFC8322] conform JSON file and is used to dissect +// documents by certain criteria +// and sets the "message" field value +// of the "Requirement" struct as a result of that. +func (r *rolieCategoryReporter) report(p *processor, domain *Domain) { + // TODO +} + func (r *integrityReporter) report(p *processor, domain *Domain) { req := r.requirement(domain) if !p.badIntegrities.used() { @@ -306,3 +354,25 @@ func (r *publicPGPKeyReporter) report(p *processor, domain *Domain) { p.keys.CountEntities())) } } + +// report tests whether a CSAF aggregator JSON schema conform +// aggregator.json exists without being adjacent to a +// provider-metadata.json +func (r *listReporter) report(p *processor, domain *Domain) { + // TODO +} + +// report tests whether the aggregator.json lists at least +// two disjoint issuing parties. TODO: reevaluate phrasing (Req 7.1.22) +func (r *hasTwoReporter) report(p *processor, domain *Domain) { + // TODO +} + +// report tests whether the CSAF documents of each issuing mirrored party +// is in a different folder, which are adjacent to the aggregator.json and +// if the folder name is retrieved from the name of the issuing authority. +// It also tests whether each folder has a provider-metadata.json for their +// party and provides ROLIE feed documents. +func (r *mirrorReporter) report(p *processor, domain *Domain) { + // TODO +} From 9ac902347ca1925e042712f2636af3876cae172f Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 13:54:21 +0200 Subject: [PATCH 10/14] Fix revive --- cmd/csaf_checker/reporters.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/csaf_checker/reporters.go b/cmd/csaf_checker/reporters.go index 1d3100f..4b0dccf 100644 --- a/cmd/csaf_checker/reporters.go +++ b/cmd/csaf_checker/reporters.go @@ -126,7 +126,7 @@ func (r *tlsReporter) report(p *processor, domain *Domain) { // report tests if a document labeled TLP:WHITE // is freely accessible and sets the "message" field value // of the "Requirement" struct as a result of that. -func (r *tlpWhiteReporter) report(p *processor, domain *Domain) { +func (r *tlpWhiteReporter) report(_ *processor, _ *Domain) { // TODO } @@ -134,7 +134,7 @@ func (r *tlpWhiteReporter) report(p *processor, domain *Domain) { // or TLP:RED is access protected // and sets the "message" field value // of the "Requirement" struct as a result of that. -func (r *tlpAmberRedReporter) report(p *processor, domain *Domain) { +func (r *tlpAmberRedReporter) report(_ *processor, _ *Domain) { // TODO } @@ -296,7 +296,7 @@ func (r *directoryListingsReporter) report(p *processor, domain *Domain) { // given TLP level and whether any of the TLP levels // TLP:WHITE, TLP:GREEN or unlabeled exists and sets the "message" field value // of the "Requirement" struct as a result of that. -func (r *rolieFeedReporter) report(p *processor, domain *Domain) { +func (r *rolieFeedReporter) report(_ *processor, _ *Domain) { // TODO } @@ -304,7 +304,7 @@ func (r *rolieFeedReporter) report(p *processor, domain *Domain) { // whether it is a [RFC8322] conform JSON file that lists the // ROLIE feed documents and sets the "message" field value // of the "Requirement" struct as a result of that. -func (r *rolieServiceReporter) report(p *processor, domain *Domain) { +func (r *rolieServiceReporter) report(_ *processor, _ *Domain) { // TODO } @@ -313,7 +313,7 @@ func (r *rolieServiceReporter) report(p *processor, domain *Domain) { // documents by certain criteria // and sets the "message" field value // of the "Requirement" struct as a result of that. -func (r *rolieCategoryReporter) report(p *processor, domain *Domain) { +func (r *rolieCategoryReporter) report(_ *processor, _ *Domain) { // TODO } @@ -358,13 +358,13 @@ func (r *publicPGPKeyReporter) report(p *processor, domain *Domain) { // report tests whether a CSAF aggregator JSON schema conform // aggregator.json exists without being adjacent to a // provider-metadata.json -func (r *listReporter) report(p *processor, domain *Domain) { +func (r *listReporter) report(_ *processor, _ *Domain) { // TODO } // report tests whether the aggregator.json lists at least // two disjoint issuing parties. TODO: reevaluate phrasing (Req 7.1.22) -func (r *hasTwoReporter) report(p *processor, domain *Domain) { +func (r *hasTwoReporter) report(_ *processor, _ *Domain) { // TODO } @@ -373,6 +373,6 @@ func (r *hasTwoReporter) report(p *processor, domain *Domain) { // if the folder name is retrieved from the name of the issuing authority. // It also tests whether each folder has a provider-metadata.json for their // party and provides ROLIE feed documents. -func (r *mirrorReporter) report(p *processor, domain *Domain) { +func (r *mirrorReporter) report(_ *processor, _ *Domain) { // TODO } From 068a94235c86b14f02ea51d9a475adbf649bb52e Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 14:01:27 +0200 Subject: [PATCH 11/14] Add PMD loading errors to bad provider metadata report. --- cmd/csaf_checker/processor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index c928814..ed0ab07 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -1121,7 +1121,10 @@ func (p *processor) checkProviderMetadata(domain string) bool { lpmd := loader.Load(domain) - // TODO: Sort results into reports. + for i := range lpmd.Messages { + // TODO: Filter depending on the role. + p.badProviderMetadata.error(lpmd.Messages[i].Message) + } if !lpmd.Valid() { p.badProviderMetadata.error("No valid provider-metadata.json found.") From 150db4d31b855e260139499dea3eb8352fa1f24e Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer Date: Mon, 15 May 2023 14:12:16 +0200 Subject: [PATCH 12/14] Add new reporters to list of reporters in csaf_checker/main.go --- cmd/csaf_checker/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index c1bcd03..6d92900 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -146,8 +146,8 @@ var reporters = [23]reporter{ &validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}}, &filenameReporter{baseReporter{num: 2, description: "Filename"}}, &tlsReporter{baseReporter{num: 3, description: "TLS"}}, - nil, // TODO: Add 4: TLP:WHITE - nil, // TODO: Add 5: TLP:AMBER and TLP:RED + &tlpWhiteReporter{baseReporter{num: 4, description: "TLP:WHITE"}}, + &tlpAmberRedReporter{baseReporter{num: 5, description: "TLP:AMBER and TLP:RED"}}, &redirectsReporter{baseReporter{num: 6, description: "Redirects"}}, &providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}}, &securityReporter{baseReporter{num: 8, description: "security.txt"}}, @@ -157,15 +157,15 @@ var reporters = [23]reporter{ &indexReporter{baseReporter{num: 12, description: "index.txt"}}, &changesReporter{baseReporter{num: 13, description: "changes.csv"}}, &directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}}, - nil, // TODO: Add 15: ROLIE feed - nil, // TODO: Add 16: ROLIE service document - nil, // TODO: Add 17: ROLIE category document + &rolieFeedReporter{baseReporter{num: 15, description: "ROLIE feed"}}, + &rolieServiceReporter{baseReporter{num: 16, description: "ROLIE service document"}}, + &rolieCategoryReporter{baseReporter{num: 17, description: "ROLIE category document"}}, &integrityReporter{baseReporter{num: 18, description: "Integrity"}}, &signaturesReporter{baseReporter{num: 19, description: "Signatures"}}, &publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}}, - nil, // TODO: Add 21: List of CSAF providers - nil, // TODO: Add 22: Two disjoint issuing parties - nil, // TODO: Add 23: Mirror + &listReporter{baseReporter{num: 21, description: "List of CSAF providers"}}, + &hasTwoReporter{baseReporter{num: 22, description: "Two disjoint issuing parties"}}, + &mirrorReporter{baseReporter{num: 23, description: "Mirror"}}, } var roleImplies = map[csaf.MetadataRole][]csaf.MetadataRole{ From a0b272a60dce5dafc16cd8a4c7996bf7746c8918 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 14:15:20 +0200 Subject: [PATCH 13/14] Deactivate TLP reporters --- cmd/csaf_checker/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 6d92900..95474de 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -179,9 +179,10 @@ func requirements(role csaf.MetadataRole) [][2]int { case csaf.MetadataRoleTrustedProvider: own = [][2]int{{18, 20}} case csaf.MetadataRoleProvider: - own = [][2]int{{5, 7}, {8, 10}, {11, 14}, {15, 17}} + // TODO: use commented numbers when TLPs should be checked. + own = [][2]int{{6 /* 5 */, 7}, {8, 10}, {11, 14}, {15, 17}} case csaf.MetadataRolePublisher: - own = [][2]int{{1, 4}} + own = [][2]int{{1, 3 /* 4 */}} } for _, base := range roleImplies[role] { own = append(own, requirements(base)...) From 1dab0cc9ff82e867a60ce98bdf16e9626f7e6b6d Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 15 May 2023 14:29:47 +0200 Subject: [PATCH 14/14] Move code to more suited place. --- cmd/csaf_checker/main.go | 68 ----------------------------------- cmd/csaf_checker/reporters.go | 68 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 95474de..a47b584 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -21,9 +21,7 @@ import ( "log" "net/http" "os" - "sort" - "github.com/csaf-poc/csaf_distribution/csaf" "github.com/csaf-poc/csaf_distribution/util" "github.com/jessevdk/go-flags" ) @@ -142,72 +140,6 @@ func writeReport(report *Report, opts *options) error { return writer(report, w) } -var reporters = [23]reporter{ - &validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}}, - &filenameReporter{baseReporter{num: 2, description: "Filename"}}, - &tlsReporter{baseReporter{num: 3, description: "TLS"}}, - &tlpWhiteReporter{baseReporter{num: 4, description: "TLP:WHITE"}}, - &tlpAmberRedReporter{baseReporter{num: 5, description: "TLP:AMBER and TLP:RED"}}, - &redirectsReporter{baseReporter{num: 6, description: "Redirects"}}, - &providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}}, - &securityReporter{baseReporter{num: 8, description: "security.txt"}}, - &wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}}, - &dnsPathReporter{baseReporter{num: 10, description: "DNS path"}}, - &oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}}, - &indexReporter{baseReporter{num: 12, description: "index.txt"}}, - &changesReporter{baseReporter{num: 13, description: "changes.csv"}}, - &directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}}, - &rolieFeedReporter{baseReporter{num: 15, description: "ROLIE feed"}}, - &rolieServiceReporter{baseReporter{num: 16, description: "ROLIE service document"}}, - &rolieCategoryReporter{baseReporter{num: 17, description: "ROLIE category document"}}, - &integrityReporter{baseReporter{num: 18, description: "Integrity"}}, - &signaturesReporter{baseReporter{num: 19, description: "Signatures"}}, - &publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}}, - &listReporter{baseReporter{num: 21, description: "List of CSAF providers"}}, - &hasTwoReporter{baseReporter{num: 22, description: "Two disjoint issuing parties"}}, - &mirrorReporter{baseReporter{num: 23, description: "Mirror"}}, -} - -var roleImplies = map[csaf.MetadataRole][]csaf.MetadataRole{ - csaf.MetadataRoleProvider: {csaf.MetadataRolePublisher}, - csaf.MetadataRoleTrustedProvider: {csaf.MetadataRoleProvider}, -} - -func requirements(role csaf.MetadataRole) [][2]int { - var own [][2]int - switch role { - case csaf.MetadataRoleTrustedProvider: - own = [][2]int{{18, 20}} - case csaf.MetadataRoleProvider: - // TODO: use commented numbers when TLPs should be checked. - own = [][2]int{{6 /* 5 */, 7}, {8, 10}, {11, 14}, {15, 17}} - case csaf.MetadataRolePublisher: - own = [][2]int{{1, 3 /* 4 */}} - } - for _, base := range roleImplies[role] { - own = append(own, requirements(base)...) - } - return own -} - -// buildReporters initializes each report by assigning a number and description to it. -// It returns an array of the reporter interface type. -func buildReporters(role csaf.MetadataRole) []reporter { - var reps []reporter - reqs := requirements(role) - // sort to have them ordered by there number. - sort.Slice(reqs, func(i, j int) bool { return reqs[i][0] < reqs[j][0] }) - for _, req := range reqs { - from, to := req[0]-1, req[1]-1 - for i := from; i <= to; i++ { - if rep := reporters[i]; rep != nil { - reps = append(reps, rep) - } - } - } - return reps -} - // run uses a processor to check all the given domains or direct urls // and generates a report. func run(opts *options, domains []string) (*Report, error) { diff --git a/cmd/csaf_checker/reporters.go b/cmd/csaf_checker/reporters.go index 4b0dccf..fc15f70 100644 --- a/cmd/csaf_checker/reporters.go +++ b/cmd/csaf_checker/reporters.go @@ -12,6 +12,8 @@ import ( "fmt" "sort" "strings" + + "github.com/csaf-poc/csaf_distribution/csaf" ) type ( @@ -44,6 +46,72 @@ type ( mirrorReporter struct{ baseReporter } ) +var reporters = [23]reporter{ + &validReporter{baseReporter{num: 1, description: "Valid CSAF documents"}}, + &filenameReporter{baseReporter{num: 2, description: "Filename"}}, + &tlsReporter{baseReporter{num: 3, description: "TLS"}}, + &tlpWhiteReporter{baseReporter{num: 4, description: "TLP:WHITE"}}, + &tlpAmberRedReporter{baseReporter{num: 5, description: "TLP:AMBER and TLP:RED"}}, + &redirectsReporter{baseReporter{num: 6, description: "Redirects"}}, + &providerMetadataReport{baseReporter{num: 7, description: "provider-metadata.json"}}, + &securityReporter{baseReporter{num: 8, description: "security.txt"}}, + &wellknownMetadataReporter{baseReporter{num: 9, description: "/.well-known/csaf/provider-metadata.json"}}, + &dnsPathReporter{baseReporter{num: 10, description: "DNS path"}}, + &oneFolderPerYearReport{baseReporter{num: 11, description: "One folder per year"}}, + &indexReporter{baseReporter{num: 12, description: "index.txt"}}, + &changesReporter{baseReporter{num: 13, description: "changes.csv"}}, + &directoryListingsReporter{baseReporter{num: 14, description: "Directory listings"}}, + &rolieFeedReporter{baseReporter{num: 15, description: "ROLIE feed"}}, + &rolieServiceReporter{baseReporter{num: 16, description: "ROLIE service document"}}, + &rolieCategoryReporter{baseReporter{num: 17, description: "ROLIE category document"}}, + &integrityReporter{baseReporter{num: 18, description: "Integrity"}}, + &signaturesReporter{baseReporter{num: 19, description: "Signatures"}}, + &publicPGPKeyReporter{baseReporter{num: 20, description: "Public OpenPGP Key"}}, + &listReporter{baseReporter{num: 21, description: "List of CSAF providers"}}, + &hasTwoReporter{baseReporter{num: 22, description: "Two disjoint issuing parties"}}, + &mirrorReporter{baseReporter{num: 23, description: "Mirror"}}, +} + +var roleImplies = map[csaf.MetadataRole][]csaf.MetadataRole{ + csaf.MetadataRoleProvider: {csaf.MetadataRolePublisher}, + csaf.MetadataRoleTrustedProvider: {csaf.MetadataRoleProvider}, +} + +func requirements(role csaf.MetadataRole) [][2]int { + var own [][2]int + switch role { + case csaf.MetadataRoleTrustedProvider: + own = [][2]int{{18, 20}} + case csaf.MetadataRoleProvider: + // TODO: use commented numbers when TLPs should be checked. + own = [][2]int{{6 /* 5 */, 7}, {8, 10}, {11, 14}, {15, 17}} + case csaf.MetadataRolePublisher: + own = [][2]int{{1, 3 /* 4 */}} + } + for _, base := range roleImplies[role] { + own = append(own, requirements(base)...) + } + return own +} + +// buildReporters initializes each report by assigning a number and description to it. +// It returns an array of the reporter interface type. +func buildReporters(role csaf.MetadataRole) []reporter { + var reps []reporter + reqs := requirements(role) + // sort to have them ordered by there number. + sort.Slice(reqs, func(i, j int) bool { return reqs[i][0] < reqs[j][0] }) + for _, req := range reqs { + from, to := req[0]-1, req[1]-1 + for i := from; i <= to; i++ { + if rep := reporters[i]; rep != nil { + reps = append(reps, rep) + } + } + } + return reps +} + func (bc *baseReporter) requirement(domain *Domain) *Requirement { req := &Requirement{ Num: bc.num,