From 73aef07063a701b54bae8ff5be6262a9dbf73dcb Mon Sep 17 00:00:00 2001 From: "Kunz, Immanuel" Date: Mon, 22 Apr 2024 17:48:11 +0200 Subject: [PATCH 1/6] add enumerate function to ProviderMetadataLoader --- csaf/providermetaloader.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 62e8876..2b2e85f 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -108,6 +108,42 @@ func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader { } } +func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMetadata { + + // Our array of PMDs to be found + var resPMDs []*LoadedProviderMetadata + + // TODO check direct path? + + // First try the well-known path. + wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" + + wellknownResult := pmdl.loadFromURL(wellknownURL) + + // Validate the candidate and add to the result array + if wellknownResult.Valid() { + resPMDs = append(resPMDs, wellknownResult) + } + + // Next load the PMDs from security.txt + secResults := pmdl.loadFromSecurity(domain) + + for _, result := range secResults { + if result.Valid() { + resPMDs = append(resPMDs, result) + } + } + + // According to the spec, only if no PMDs have been found, should the DNS URL be used + if len(resPMDs) > 0 { + return resPMDs + } else { + dnsURL := "https://csaf.data.security." + domain + return []*LoadedProviderMetadata{pmdl.loadFromURL(dnsURL)} + } + +} + // Load loads a provider metadata for a given path. // If the domain starts with `https://` it only attemps to load // the data from that URL. From d64aa20cee579c04b8d38b5470d5bf899141dad2 Mon Sep 17 00:00:00 2001 From: "Kunz, Immanuel" Date: Mon, 22 Apr 2024 17:53:45 +0200 Subject: [PATCH 2/6] first draft for downloader using enumerate --- cmd/csaf_downloader/downloader.go | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 7fa0c7c..86939d8 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -165,6 +165,39 @@ func httpLog(who string) func(string, string) { } } +func (d *downloader) enumerate(ctx context.Context, domain string) error { + client := d.httpClient() + + loader := csaf.NewProviderMetadataLoader(client) + + lpmd := loader.Enumerate(domain) + + if d.cfg.verbose() { + for i := range lpmd.Messages { + slog.Debug("Loading provider-metadata.json", + "domain", domain, + "message", lpmd.Messages[i].Message) + } + } + + for _, pmd := range lpmd { + if !pmd.Valid() { + return fmt.Errorf("invalid provider-metadata.json found for '%s'", domain) + } + _, err := url.Parse(pmd.URL) + if err != nil { + return fmt.Errorf("invalid URL found '%s': %v", pmd.URL, err) + } + + // TODO print + fmt.Println(pmd.URL) + fmt.Println(pmd.Document) + fmt.Println(pmd.Messages) + fmt.Println(pmd.Hash) + } + +} + func (d *downloader) download(ctx context.Context, domain string) error { client := d.httpClient() From 457d519990425ca1f77526c17a2210e4ea02b98d Mon Sep 17 00:00:00 2001 From: "Kunz, Immanuel" Date: Tue, 23 Apr 2024 19:09:22 +0200 Subject: [PATCH 3/6] minor updates to Enumerate method, integrate enumerate in cmd downloader --- cmd/csaf_downloader/downloader.go | 43 ++++++++++++++++++++----------- cmd/csaf_downloader/main.go | 5 ++++ csaf/providermetaloader.go | 15 +++++++---- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 86939d8..d8a966f 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -165,22 +165,22 @@ func httpLog(who string) func(string, string) { } } -func (d *downloader) enumerate(ctx context.Context, domain string) error { +func (d *downloader) enumerate(domain string) error { client := d.httpClient() loader := csaf.NewProviderMetadataLoader(client) lpmd := loader.Enumerate(domain) - if d.cfg.verbose() { - for i := range lpmd.Messages { - slog.Debug("Loading provider-metadata.json", - "domain", domain, - "message", lpmd.Messages[i].Message) - } - } - for _, pmd := range lpmd { + if d.cfg.verbose() { + for i := range pmd.Messages { + slog.Debug("Enumerating provider-metadata.json", + "domain", domain, + "message", pmd.Messages[i].Message) + } + } + if !pmd.Valid() { return fmt.Errorf("invalid provider-metadata.json found for '%s'", domain) } @@ -189,13 +189,15 @@ func (d *downloader) enumerate(ctx context.Context, domain string) error { return fmt.Errorf("invalid URL found '%s': %v", pmd.URL, err) } - // TODO print - fmt.Println(pmd.URL) - fmt.Println(pmd.Document) - fmt.Println(pmd.Messages) - fmt.Println(pmd.Hash) + // print the results + fmt.Println("Found provider-metadata file under URL", pmd.URL) + doc, err := json.MarshalIndent(pmd.Document, "", " ") + if err != nil { + slog.Error("Couldn't marshal PMD document json") + } + fmt.Println(string(doc)) } - + return nil } func (d *downloader) download(ctx context.Context, domain string) error { @@ -775,3 +777,14 @@ func (d *downloader) run(ctx context.Context, domains []string) error { } return nil } + +// runEnumerate performs the enumeration of PMDs for all the given domains. +func (d *downloader) runEnumerate(domains []string) error { + defer d.stats.log() + for _, domain := range domains { + if err := d.enumerate(domain); err != nil { + return err + } + } + return nil +} diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index daff163..e54b2e1 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -41,6 +41,11 @@ func run(cfg *config, domains []string) error { d.forwarder = f } + // First, enumerate existing PMDs, then load + err = d.runEnumerate(domains) + if err != nil { + return err + } return d.run(ctx, domains) } diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 2b2e85f..9549e3d 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -45,7 +45,7 @@ const ( // WellknownSecurityMismatch indicates that the PMDs found under wellknown and // in the security do not match. WellknownSecurityMismatch - // IgnoreProviderMetadata indicates that a extra PMD was ignored. + // IgnoreProviderMetadata indicates that an extra PMD was ignored. IgnoreProviderMetadata ) @@ -113,7 +113,10 @@ func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMe // Our array of PMDs to be found var resPMDs []*LoadedProviderMetadata - // TODO check direct path? + // Check direct path + if strings.HasPrefix(domain, "https://") { + return []*LoadedProviderMetadata{pmdl.loadFromURL(domain)} + } // First try the well-known path. wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" @@ -122,11 +125,13 @@ func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMe // Validate the candidate and add to the result array if wellknownResult.Valid() { + fmt.Println("Found well known result") resPMDs = append(resPMDs, wellknownResult) } // Next load the PMDs from security.txt secResults := pmdl.loadFromSecurity(domain) + fmt.Println("Found security.txt results", len(secResults)) for _, result := range secResults { if result.Valid() { @@ -134,7 +139,7 @@ func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMe } } - // According to the spec, only if no PMDs have been found, should the DNS URL be used + // According to the spec, only if no PMDs have been found, the should DNS URL be used if len(resPMDs) > 0 { return resPMDs } else { @@ -144,8 +149,8 @@ func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMe } -// Load loads a provider metadata for a given path. -// If the domain starts with `https://` it only attemps to load +// Load loads one valid provider metadata for a given path. +// If the domain starts with `https://` it only attempts to load // the data from that URL. func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata { From 005e661479038df95134d7edd8745e3378980f96 Mon Sep 17 00:00:00 2001 From: "Kunz, Immanuel" Date: Tue, 23 Apr 2024 20:24:18 +0200 Subject: [PATCH 4/6] add config flag to use enumerate-only --- cmd/csaf_downloader/config.go | 2 ++ cmd/csaf_downloader/downloader.go | 27 +++++++++++---------------- cmd/csaf_downloader/main.go | 10 +++++----- csaf/providermetaloader.go | 5 +++-- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 367780f..00780ee 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -58,6 +58,8 @@ type config struct { IgnorePattern []string `long:"ignore_pattern" short:"i" description:"Do not download files if their URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignore_pattern"` ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"` + EnumeratePMDOnly bool `long:"enumerate_pmd_only" description:"If this flag is set to true, the donwloader will only enumerate valid provider metadata files, but not download documents" toml:"enumerate_pmd_only"` + RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL" toml:"validator"` RemoteValidatorCache string `long:"validator_cache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validator_cache"` RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"` diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index d8a966f..80c6350 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -169,9 +169,10 @@ func (d *downloader) enumerate(domain string) error { client := d.httpClient() loader := csaf.NewProviderMetadataLoader(client) - lpmd := loader.Enumerate(domain) + docs := []any{} + for _, pmd := range lpmd { if d.cfg.verbose() { for i := range pmd.Messages { @@ -181,22 +182,16 @@ func (d *downloader) enumerate(domain string) error { } } - if !pmd.Valid() { - return fmt.Errorf("invalid provider-metadata.json found for '%s'", domain) - } - _, err := url.Parse(pmd.URL) - if err != nil { - return fmt.Errorf("invalid URL found '%s': %v", pmd.URL, err) - } - - // print the results - fmt.Println("Found provider-metadata file under URL", pmd.URL) - doc, err := json.MarshalIndent(pmd.Document, "", " ") - if err != nil { - slog.Error("Couldn't marshal PMD document json") - } - fmt.Println(string(doc)) + docs = append(docs, pmd.Document) } + + // print the results + doc, err := json.MarshalIndent(docs, "", " ") + if err != nil { + slog.Error("Couldn't marshal PMD document json") + } + fmt.Println(string(doc)) + return nil } diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index e54b2e1..3a0d79f 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -41,12 +41,12 @@ func run(cfg *config, domains []string) error { d.forwarder = f } - // First, enumerate existing PMDs, then load - err = d.runEnumerate(domains) - if err != nil { - return err + if cfg.EnumeratePMDOnly { + // Enumerate only + return d.runEnumerate(domains) + } else { + return d.run(ctx, domains) } - return d.run(ctx, domains) } func main() { diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 9549e3d..1c1200d 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -14,6 +14,7 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net/http" "strings" @@ -125,13 +126,13 @@ func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMe // Validate the candidate and add to the result array if wellknownResult.Valid() { - fmt.Println("Found well known result") + slog.Debug("Found well known provider-metadata.json") resPMDs = append(resPMDs, wellknownResult) } // Next load the PMDs from security.txt secResults := pmdl.loadFromSecurity(domain) - fmt.Println("Found security.txt results", len(secResults)) + slog.Info("Found provider metadata results in security.txt", "num", len(secResults)) for _, result := range secResults { if result.Valid() { From 684770ff2ef94983be4caebada03119d6551d34c Mon Sep 17 00:00:00 2001 From: "Kunz, Immanuel" Date: Wed, 24 Apr 2024 17:53:47 +0200 Subject: [PATCH 5/6] fix typo, fix linting errors --- cmd/csaf_downloader/config.go | 2 +- cmd/csaf_downloader/main.go | 6 +++--- csaf/providermetaloader.go | 8 +++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 00780ee..e534c61 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -58,7 +58,7 @@ type config struct { IgnorePattern []string `long:"ignore_pattern" short:"i" description:"Do not download files if their URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignore_pattern"` ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"` - EnumeratePMDOnly bool `long:"enumerate_pmd_only" description:"If this flag is set to true, the donwloader will only enumerate valid provider metadata files, but not download documents" toml:"enumerate_pmd_only"` + EnumeratePMDOnly bool `long:"enumerate_pmd_only" description:"If this flag is set to true, the downloader will only enumerate valid provider metadata files, but not download documents" toml:"enumerate_pmd_only"` RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL" toml:"validator"` RemoteValidatorCache string `long:"validator_cache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validator_cache"` diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index 3a0d79f..6852217 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -41,12 +41,12 @@ func run(cfg *config, domains []string) error { d.forwarder = f } + // If the enumerate-only flag is set, enumerate found PMDs, + // else use the normal load method if cfg.EnumeratePMDOnly { - // Enumerate only return d.runEnumerate(domains) - } else { - return d.run(ctx, domains) } + return d.run(ctx, domains) } func main() { diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 1c1200d..45436b1 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -109,6 +109,9 @@ func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader { } } +// Enumerate lists all PMD files that can be found under the given domain. +// As specified in CSAF 2.0, it looks for PMDs using the well-known URL and +// the security.txt, and if no PMDs have been found, it also checks the DNS-URL. func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMetadata { // Our array of PMDs to be found @@ -143,10 +146,9 @@ func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMe // According to the spec, only if no PMDs have been found, the should DNS URL be used if len(resPMDs) > 0 { return resPMDs - } else { - dnsURL := "https://csaf.data.security." + domain - return []*LoadedProviderMetadata{pmdl.loadFromURL(dnsURL)} } + dnsURL := "https://csaf.data.security." + domain + return []*LoadedProviderMetadata{pmdl.loadFromURL(dnsURL)} } From a608cb0b17487bac2a4db37f51aba561bb9f534a Mon Sep 17 00:00:00 2001 From: "immqu immqu@users.noreply.github.com" Date: Thu, 25 Apr 2024 07:43:28 +0000 Subject: [PATCH 6/6] Apply automatic changes --- docs/csaf_aggregator.md | 12 +++++++----- docs/csaf_provider.md | 12 +----------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index 042d321..36cbe7e 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -177,7 +177,7 @@ categories document. For a more detailed explanation and examples, ```toml workers = 2 folder = "/var/csaf_aggregator" -lock_file = "/var/csaf_aggregator/run.lock" +lock_file = "/var/lock/csaf_aggregator/lock" web = "/var/csaf_aggregator/html" domain = "https://localhost:9443" rate = 10.0 @@ -187,6 +187,7 @@ insecure = true #interim_years = #passphrase = #write_indices = false +#time_range = # specification requires at least two providers (default), # to override for testing, enable: @@ -208,6 +209,7 @@ insecure = true create_service_document = true # rate = 1.5 # insecure = true +# time_range = [[providers]] name = "local-dev-provider2" @@ -217,8 +219,8 @@ insecure = true write_indices = true client_cert = "./../devca1/testclient1.crt" client_key = "./../devca1/testclient1-key.pem" -# client_passphrase = -# header = +# client_passphrase = # Limited and experimental, see downloader doc. +# header = [[providers]] name = "local-dev-provider3" @@ -226,10 +228,10 @@ insecure = true # rate = 1.8 # insecure = true write_indices = true - # If aggregator.category == "aggregator", set for an entry that should + # If aggregator.category == "aggreator", set for an entry that should # be listed in addition: category = "lister" -# ignore_pattern = [".*white.*", ".*red.*"] +# ignore_pattern = [".*white.*", ".*red.*"] ``` diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index b02165b..81a45fa 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -100,22 +100,12 @@ The following example file documents all available configuration options: #tlps = ["csaf", "white", "amber", "green", "red"] # Make the provider create a ROLIE service document. -#create_service_document = true +#create_service_document = false # Make the provider create a ROLIE category document from a list of strings. # If a list item starts with `expr:` # the rest of the string is used as a JsonPath expression # to extract a string from the incoming advisories. -# If the result of the expression is a string this string -# is used. If the result is an array each element of -# this array is tested if it is a string or an array. -# If this test fails the expression fails. If the -# test succeeds the rules are applied recursively to -# collect all strings in the result. -# Suggested expressions are: -# - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name" -# - CVEs: "expr:$.vulnerabilities[*].cve" -# - CWEs: "expr:$.vulnerabilities[*].cwe.id" # Strings not starting with `expr:` are taken verbatim. # By default no category documents are created. # This example provides an overview over the syntax,