Merge pull request #536 from immqu/main

Print provider-metadata.json files per domain

cmd/csaf_downloader/config.go

@@ -58,6 +58,8 @@ type config struct {
 	IgnorePattern        []string          `long:"ignore_pattern" short:"i" description:"Do not download files if their URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignore_pattern"`
 	ExtraHeader          http.Header       `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"`
 
+	EnumeratePMDOnly bool `long:"enumerate_pmd_only" description:"If this flag is set to true, the downloader will only enumerate valid provider metadata files, but not download documents" toml:"enumerate_pmd_only"`
+
 	RemoteValidator        string   `long:"validator" description:"URL to validate documents remotely" value-name:"URL" toml:"validator"`
 	RemoteValidatorCache   string   `long:"validator_cache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validator_cache"`
 	RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"`

cmd/csaf_downloader/downloader.go

@@ -165,6 +165,36 @@ func httpLog(who string) func(string, string) {
 	}
 }
 
+func (d *downloader) enumerate(domain string) error {
+	client := d.httpClient()
+
+	loader := csaf.NewProviderMetadataLoader(client)
+	lpmd := loader.Enumerate(domain)
+
+	docs := []any{}
+
+	for _, pmd := range lpmd {
+		if d.cfg.verbose() {
+			for i := range pmd.Messages {
+				slog.Debug("Enumerating provider-metadata.json",
+					"domain", domain,
+					"message", pmd.Messages[i].Message)
+			}
+		}
+
+		docs = append(docs, pmd.Document)
+	}
+
+	// print the results
+	doc, err := json.MarshalIndent(docs, "", "  ")
+	if err != nil {
+		slog.Error("Couldn't marshal PMD document json")
+	}
+	fmt.Println(string(doc))
+
+	return nil
+}
+
 func (d *downloader) download(ctx context.Context, domain string) error {
 	client := d.httpClient()
 
@@ -742,3 +772,14 @@ func (d *downloader) run(ctx context.Context, domains []string) error {
 	}
 	return nil
 }
+
+// runEnumerate performs the enumeration of PMDs for all the given domains.
+func (d *downloader) runEnumerate(domains []string) error {
+	defer d.stats.log()
+	for _, domain := range domains {
+		if err := d.enumerate(domain); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/csaf_downloader/main.go

@@ -41,6 +41,11 @@ func run(cfg *config, domains []string) error {
 		d.forwarder = f
 	}
 
+	// If the enumerate-only flag is set, enumerate found PMDs,
+	// else use the normal load method
+	if cfg.EnumeratePMDOnly {
+		return d.runEnumerate(domains)
+	}
 	return d.run(ctx, domains)
 }
 

csaf/providermetaloader.go

@@ -14,6 +14,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"strings"
 
@@ -45,7 +46,7 @@ const (
 	// WellknownSecurityMismatch indicates that the PMDs found under wellknown and
 	// in the security do not match.
 	WellknownSecurityMismatch
-	// IgnoreProviderMetadata indicates that a extra PMD was ignored.
+	// IgnoreProviderMetadata indicates that an extra PMD was ignored.
 	IgnoreProviderMetadata
 )
 
@@ -108,8 +109,51 @@ func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader {
 	}
 }
 
-// Load loads a provider metadata for a given path.
-// If the domain starts with `https://` it only attemps to load
+// Enumerate lists all PMD files that can be found under the given domain.
+// As specified in CSAF 2.0, it looks for PMDs using the well-known URL and
+// the security.txt, and if no PMDs have been found, it also checks the DNS-URL.
+func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMetadata {
+
+	// Our array of PMDs to be found
+	var resPMDs []*LoadedProviderMetadata
+
+	// Check direct path
+	if strings.HasPrefix(domain, "https://") {
+		return []*LoadedProviderMetadata{pmdl.loadFromURL(domain)}
+	}
+
+	// First try the well-known path.
+	wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
+
+	wellknownResult := pmdl.loadFromURL(wellknownURL)
+
+	// Validate the candidate and add to the result array
+	if wellknownResult.Valid() {
+		slog.Debug("Found well known provider-metadata.json")
+		resPMDs = append(resPMDs, wellknownResult)
+	}
+
+	// Next load the PMDs from security.txt
+	secResults := pmdl.loadFromSecurity(domain)
+	slog.Info("Found provider metadata results in security.txt", "num", len(secResults))
+
+	for _, result := range secResults {
+		if result.Valid() {
+			resPMDs = append(resPMDs, result)
+		}
+	}
+
+	// According to the spec, only if no PMDs have been found, the should DNS URL be used
+	if len(resPMDs) > 0 {
+		return resPMDs
+	}
+	dnsURL := "https://csaf.data.security." + domain
+	return []*LoadedProviderMetadata{pmdl.loadFromURL(dnsURL)}
+
+}
+
+// Load loads one valid provider metadata for a given path.
+// If the domain starts with `https://` it only attempts to load
 // the data from that URL.
 func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata {
 

docs/csaf_aggregator.md

@@ -177,7 +177,7 @@ categories document. For a more detailed explanation and examples,
 ```toml
 workers = 2
 folder = "/var/csaf_aggregator"
-lock_file = "/var/csaf_aggregator/run.lock"
+lock_file = "/var/lock/csaf_aggregator/lock"
 web = "/var/csaf_aggregator/html"
 domain = "https://localhost:9443"
 rate = 10.0
@@ -187,6 +187,7 @@ insecure = true
 #interim_years =
 #passphrase =
 #write_indices = false
+#time_range =
 
 # specification requires at least two providers (default),
 # to override for testing, enable:
@@ -208,6 +209,7 @@ insecure = true
   create_service_document = true
 #  rate = 1.5
 #  insecure = true
+#  time_range =
 
 [[providers]]
   name = "local-dev-provider2"
@@ -217,8 +219,8 @@ insecure = true
   write_indices = true
   client_cert = "./../devca1/testclient1.crt"
   client_key = "./../devca1/testclient1-key.pem"
-#  client_passphrase =
-# header =
+#  client_passphrase = # Limited and experimental, see downloader doc.
+#  header =
 
 [[providers]]
   name = "local-dev-provider3"
@@ -226,10 +228,10 @@ insecure = true
 #  rate = 1.8
 #  insecure = true
   write_indices = true
-  # If aggregator.category == "aggregator", set for an entry that should
+  # If aggregator.category == "aggreator", set for an entry that should
   # be listed in addition:
   category = "lister"
-#  ignore_pattern = [".*white.*", ".*red.*"]
+# ignore_pattern = [".*white.*", ".*red.*"]
 ```
 <!-- MARKDOWN-AUTO-DOCS:END -->
 

docs/csaf_provider.md

@@ -100,22 +100,12 @@ The following example file documents all available configuration options:
 #tlps = ["csaf", "white", "amber", "green", "red"]
 
 # Make the provider create a ROLIE service document.
-#create_service_document = true
+#create_service_document = false
 
 # Make the provider create a ROLIE category document from a list of strings.
 # If a list item starts with `expr:`
 #   the rest of the string is used as a JsonPath expression
 #   to extract a string from the incoming advisories.
-#   If the result of the expression is a string this string
-#   is used. If the result is an array each element of
-#   this array is tested if it is a string or an array.
-#   If this test fails the expression fails. If the
-#   test succeeds the rules are applied recursively to
-#   collect all strings in the result.
-#   Suggested expressions are:
-#   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name"
-#   - CVEs: "expr:$.vulnerabilities[*].cve"
-#   - CWEs: "expr:$.vulnerabilities[*].cwe.id"
 # Strings not starting with `expr:` are taken verbatim.
 # By default no category documents are created.
 # This example provides an overview over the syntax,