add config flag to use enumerate-only

cmd/csaf_downloader/config.go

@@ -58,6 +58,8 @@ type config struct {
 	IgnorePattern        []string          `long:"ignore_pattern" short:"i" description:"Do not download files if their URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignore_pattern"`
 	ExtraHeader          http.Header       `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"`
 
+	EnumeratePMDOnly bool `long:"enumerate_pmd_only" description:"If this flag is set to true, the donwloader will only enumerate valid provider metadata files, but not download documents" toml:"enumerate_pmd_only"`
+
 	RemoteValidator        string   `long:"validator" description:"URL to validate documents remotely" value-name:"URL" toml:"validator"`
 	RemoteValidatorCache   string   `long:"validator_cache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validator_cache"`
 	RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"`

cmd/csaf_downloader/downloader.go

@@ -169,9 +169,10 @@ func (d *downloader) enumerate(domain string) error {
 	client := d.httpClient()
 
 	loader := csaf.NewProviderMetadataLoader(client)
-
 	lpmd := loader.Enumerate(domain)
 
+	docs := []any{}
+
 	for _, pmd := range lpmd {
 		if d.cfg.verbose() {
 			for i := range pmd.Messages {
@@ -181,22 +182,16 @@ func (d *downloader) enumerate(domain string) error {
 			}
 		}
 
-		if !pmd.Valid() {
-			return fmt.Errorf("invalid provider-metadata.json found for '%s'", domain)
-		}
-		_, err := url.Parse(pmd.URL)
-		if err != nil {
-			return fmt.Errorf("invalid URL found '%s': %v", pmd.URL, err)
-		}
-
-		// print the results
-		fmt.Println("Found provider-metadata file under URL", pmd.URL)
-		doc, err := json.MarshalIndent(pmd.Document, "", "  ")
-		if err != nil {
-			slog.Error("Couldn't marshal PMD document json")
-		}
-		fmt.Println(string(doc))
+		docs = append(docs, pmd.Document)
 	}
+
+	// print the results
+	doc, err := json.MarshalIndent(docs, "", "  ")
+	if err != nil {
+		slog.Error("Couldn't marshal PMD document json")
+	}
+	fmt.Println(string(doc))
+
 	return nil
 }
 

cmd/csaf_downloader/main.go

@@ -41,12 +41,12 @@ func run(cfg *config, domains []string) error {
 		d.forwarder = f
 	}
 
-	// First, enumerate existing PMDs, then load
-	err = d.runEnumerate(domains)
-	if err != nil {
-		return err
+	if cfg.EnumeratePMDOnly {
+		// Enumerate only
+		return d.runEnumerate(domains)
+	} else {
+		return d.run(ctx, domains)
 	}
-	return d.run(ctx, domains)
 }
 
 func main() {

csaf/providermetaloader.go

@@ -14,6 +14,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"strings"
 
@@ -125,13 +126,13 @@ func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMe
 
 	// Validate the candidate and add to the result array
 	if wellknownResult.Valid() {
-		fmt.Println("Found well known result")
+		slog.Debug("Found well known provider-metadata.json")
 		resPMDs = append(resPMDs, wellknownResult)
 	}
 
 	// Next load the PMDs from security.txt
 	secResults := pmdl.loadFromSecurity(domain)
-	fmt.Println("Found security.txt results", len(secResults))
+	slog.Info("Found provider metadata results in security.txt", "num", len(secResults))
 
 	for _, result := range secResults {
 		if result.Valid() {