diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 39a4d05..470d46d 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -58,6 +58,8 @@ type config struct { IgnorePattern []string `long:"ignore_pattern" short:"i" description:"Do not download files if their URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignore_pattern"` ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"` + EnumeratePMDOnly bool `long:"enumerate_pmd_only" description:"If this flag is set to true, the downloader will only enumerate valid provider metadata files, but not download documents" toml:"enumerate_pmd_only"` + RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL" toml:"validator"` RemoteValidatorCache string `long:"validator_cache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validator_cache"` RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"` diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 38203bf..20038a6 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -165,6 +165,36 @@ func httpLog(who string) func(string, string) { } } +func (d *downloader) enumerate(domain string) error { + client := d.httpClient() + + loader := csaf.NewProviderMetadataLoader(client) + lpmd := loader.Enumerate(domain) + + docs := []any{} + + for _, pmd := range lpmd { + if d.cfg.verbose() { + for i := range pmd.Messages { + slog.Debug("Enumerating provider-metadata.json", + "domain", domain, + "message", pmd.Messages[i].Message) + } + } + + docs = append(docs, pmd.Document) + } + + // print the results + doc, err := json.MarshalIndent(docs, "", " ") + if err != nil { + slog.Error("Couldn't marshal PMD document json") + } + fmt.Println(string(doc)) + + return nil +} + func (d *downloader) download(ctx context.Context, domain string) error { client := d.httpClient() @@ -742,3 +772,14 @@ func (d *downloader) run(ctx context.Context, domains []string) error { } return nil } + +// runEnumerate performs the enumeration of PMDs for all the given domains. +func (d *downloader) runEnumerate(domains []string) error { + defer d.stats.log() + for _, domain := range domains { + if err := d.enumerate(domain); err != nil { + return err + } + } + return nil +} diff --git a/cmd/csaf_downloader/main.go b/cmd/csaf_downloader/main.go index 9364b88..aa9df6a 100644 --- a/cmd/csaf_downloader/main.go +++ b/cmd/csaf_downloader/main.go @@ -41,6 +41,11 @@ func run(cfg *config, domains []string) error { d.forwarder = f } + // If the enumerate-only flag is set, enumerate found PMDs, + // else use the normal load method + if cfg.EnumeratePMDOnly { + return d.runEnumerate(domains) + } return d.run(ctx, domains) } diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 22e0a73..203f2b3 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -14,6 +14,7 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net/http" "strings" @@ -45,7 +46,7 @@ const ( // WellknownSecurityMismatch indicates that the PMDs found under wellknown and // in the security do not match. WellknownSecurityMismatch - // IgnoreProviderMetadata indicates that a extra PMD was ignored. + // IgnoreProviderMetadata indicates that an extra PMD was ignored. IgnoreProviderMetadata ) @@ -108,8 +109,51 @@ func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader { } } -// Load loads a provider metadata for a given path. -// If the domain starts with `https://` it only attemps to load +// Enumerate lists all PMD files that can be found under the given domain. +// As specified in CSAF 2.0, it looks for PMDs using the well-known URL and +// the security.txt, and if no PMDs have been found, it also checks the DNS-URL. +func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMetadata { + + // Our array of PMDs to be found + var resPMDs []*LoadedProviderMetadata + + // Check direct path + if strings.HasPrefix(domain, "https://") { + return []*LoadedProviderMetadata{pmdl.loadFromURL(domain)} + } + + // First try the well-known path. + wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" + + wellknownResult := pmdl.loadFromURL(wellknownURL) + + // Validate the candidate and add to the result array + if wellknownResult.Valid() { + slog.Debug("Found well known provider-metadata.json") + resPMDs = append(resPMDs, wellknownResult) + } + + // Next load the PMDs from security.txt + secResults := pmdl.loadFromSecurity(domain) + slog.Info("Found provider metadata results in security.txt", "num", len(secResults)) + + for _, result := range secResults { + if result.Valid() { + resPMDs = append(resPMDs, result) + } + } + + // According to the spec, only if no PMDs have been found, the should DNS URL be used + if len(resPMDs) > 0 { + return resPMDs + } + dnsURL := "https://csaf.data.security." + domain + return []*LoadedProviderMetadata{pmdl.loadFromURL(dnsURL)} + +} + +// Load loads one valid provider metadata for a given path. +// If the domain starts with `https://` it only attempts to load // the data from that URL. func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata { diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index 042d321..36cbe7e 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -177,7 +177,7 @@ categories document. For a more detailed explanation and examples, ```toml workers = 2 folder = "/var/csaf_aggregator" -lock_file = "/var/csaf_aggregator/run.lock" +lock_file = "/var/lock/csaf_aggregator/lock" web = "/var/csaf_aggregator/html" domain = "https://localhost:9443" rate = 10.0 @@ -187,6 +187,7 @@ insecure = true #interim_years = #passphrase = #write_indices = false +#time_range = # specification requires at least two providers (default), # to override for testing, enable: @@ -208,6 +209,7 @@ insecure = true create_service_document = true # rate = 1.5 # insecure = true +# time_range = [[providers]] name = "local-dev-provider2" @@ -217,8 +219,8 @@ insecure = true write_indices = true client_cert = "./../devca1/testclient1.crt" client_key = "./../devca1/testclient1-key.pem" -# client_passphrase = -# header = +# client_passphrase = # Limited and experimental, see downloader doc. +# header = [[providers]] name = "local-dev-provider3" @@ -226,10 +228,10 @@ insecure = true # rate = 1.8 # insecure = true write_indices = true - # If aggregator.category == "aggregator", set for an entry that should + # If aggregator.category == "aggreator", set for an entry that should # be listed in addition: category = "lister" -# ignore_pattern = [".*white.*", ".*red.*"] +# ignore_pattern = [".*white.*", ".*red.*"] ``` diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index b02165b..81a45fa 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -100,22 +100,12 @@ The following example file documents all available configuration options: #tlps = ["csaf", "white", "amber", "green", "red"] # Make the provider create a ROLIE service document. -#create_service_document = true +#create_service_document = false # Make the provider create a ROLIE category document from a list of strings. # If a list item starts with `expr:` # the rest of the string is used as a JsonPath expression # to extract a string from the incoming advisories. -# If the result of the expression is a string this string -# is used. If the result is an array each element of -# this array is tested if it is a string or an array. -# If this test fails the expression fails. If the -# test succeeds the rules are applied recursively to -# collect all strings in the result. -# Suggested expressions are: -# - vendor, product family and product names: "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name" -# - CVEs: "expr:$.vulnerabilities[*].cve" -# - CWEs: "expr:$.vulnerabilities[*].cwe.id" # Strings not starting with `expr:` are taken verbatim. # By default no category documents are created. # This example provides an overview over the syntax,