Merge branch 'main' into publisher-in-report

cmd/csaf_checker/links.go

@@ -25,7 +25,11 @@ type (
 	pages map[string]*pageContent
 )
 
-func (pgs pages) listed(path string, pro *processor) (bool, error) {
+func (pgs pages) listed(
+	path string,
+	pro *processor,
+	badDirs map[string]struct{},
+) (bool, error) {
 	pathURL, err := url.Parse(path)
 	if err != nil {
 		return false, err
@@ -50,6 +54,10 @@ func (pgs pages) listed(path string, pro *processor) (bool, error) {
 		return false, err
 	}
 
+	if _, ok := badDirs[base]; ok {
+		return false, errContinue
+	}
+
 	// load page
 	client := pro.httpClient()
 	pro.checkTLS(base)
@@ -59,11 +67,13 @@ func (pgs pages) listed(path string, pro *processor) (bool, error) {
 
 	if err != nil {
 		pro.badDirListings.error("Fetching %s failed: %v", base, err)
+		badDirs[base] = struct{}{}
 		return false, errContinue
 	}
 	if res.StatusCode != http.StatusOK {
 		pro.badDirListings.error("Fetching %s failed. Status code %d (%s)",
 			base, res.StatusCode, res.Status)
+		badDirs[base] = struct{}{}
 		return false, errContinue
 	}
 

cmd/csaf_checker/main.go

@@ -34,6 +34,7 @@ type options struct {
 	Version    bool     `long:"version" description:"Display version of the binary"`
 	Verbose    bool     `long:"verbose" short:"v" description:"Verbose output"`
 	Rate       *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
+	Years      *uint    `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"`
 }
 
 func errCheck(err error) {

cmd/csaf_checker/processor.go

@@ -42,6 +42,7 @@ type topicMessages []Message
 type processor struct {
 	opts      *options
 	client    util.Client
+	ageAccept func(time.Time) bool
 
 	redirects      map[string][]string
 	noneTLS        map[string]struct{}
@@ -159,6 +160,17 @@ func newProcessor(opts *options) *processor {
 		opts:           opts,
 		alreadyChecked: map[string]whereType{},
 		expr:           util.NewPathEval(),
+		ageAccept:      ageAccept(opts),
+	}
+}
+
+func ageAccept(opts *options) func(time.Time) bool {
+	if opts.Years == nil {
+		return nil
+	}
+	good := time.Now().AddDate(-int(*opts.Years), 0, 0)
+	return func(t time.Time) bool {
+		return !t.Before(good)
 	}
 }
 
@@ -407,6 +419,22 @@ func (p *processor) integrity(
 			continue
 		}
 		p.checkTLS(u)
+
+		var folderYear *int
+
+		if m := yearFromURL.FindStringSubmatch(u); m != nil {
+			year, _ := strconv.Atoi(m[1])
+			// Check if we are in checking time interval.
+			if p.ageAccept != nil && !p.ageAccept(
+				time.Date(
+					year, 12, 31, // Assume last day og year.
+					23, 59, 59, 0, // 23:59:59
+					time.UTC)) {
+				continue
+			}
+			folderYear = &year
+		}
+
 		res, err := client.Get(u)
 		if err != nil {
 			lg(ErrorType, "Fetching %s failed: %v.", u, err)
@@ -455,9 +483,9 @@ func (p *processor) integrity(
 		} else if d, err := time.Parse(time.RFC3339, text); err != nil {
 			p.badFolders.error(
 				"Parsing 'initial_release_date' as RFC3339 failed in %s: %v", u, err)
-		} else if m := yearFromURL.FindStringSubmatch(u); m == nil {
+		} else if folderYear == nil {
 			p.badFolders.error("No year folder found in %s", u)
-		} else if year, _ := strconv.Atoi(m[1]); d.UTC().Year() != year {
+		} else if d.UTC().Year() != *folderYear {
 			p.badFolders.error("%s should be in folder %d", u, d.UTC().Year())
 		}
 
@@ -619,6 +647,13 @@ func (p *processor) processROLIEFeed(feed string) error {
 
 	rfeed.Entries(func(entry *csaf.Entry) {
 
+		// Filter if we have date checking.
+		if p.ageAccept != nil {
+			if pub := time.Time(entry.Published); !pub.IsZero() && !p.ageAccept(pub) {
+				return
+			}
+		}
+
 		var url, sha256, sha512, sign string
 
 		for i := range entry.Link {
@@ -790,6 +825,10 @@ func (p *processor) checkChanges(base string, mask whereType) error {
 			if err != nil {
 				return nil, nil, err
 			}
+			// Apply date range filtering.
+			if p.ageAccept != nil && !p.ageAccept(t) {
+				continue
+			}
 			times, files =
 				append(times, t),
 				append(files, csaf.PlainAdvisoryFile(r[pathColumn]))
@@ -945,8 +984,10 @@ func (p *processor) checkListing(string) error {
 
 	var unlisted []string
 
+	badDirs := map[string]struct{}{}
+
 	for f := range p.alreadyChecked {
-		found, err := pgs.listed(f, p)
+		found, err := pgs.listed(f, p, badDirs)
 		if err != nil && err != errContinue {
 			return err
 		}

csaf/util.go

@@ -223,20 +223,8 @@ func LoadProviderMetadataForDomain(
 	// Last resort: fall back to DNS.
 	dnsURL := "https://csaf.data.security." + domain
 	dnsResult := LoadProviderMetadataFromURL(client, dnsURL)
-
+	lg(dnsResult, dnsURL)
-	if dnsResult == nil {
-		logging("%s not found.", dnsURL)
-	} else if len(dnsResult.Messages) > 0 {
-		for _, msg := range dnsResult.Messages {
-			logging(msg)
-		}
-	} else {
-		// DNS seems to be okay.
 	return dnsResult
-	}
-
-	// We failed all.
-	return nil
 }
 
 // ExtractProviderURL extracts URLs of provider metadata.

docs/csaf_checker.md

@@ -3,7 +3,7 @@
 ### Usage
 
 ```
-  csaf_checker [OPTIONS]
+csaf_checker [OPTIONS]
 
 Application Options:
   -o, --output=REPORT-FILE       File name of the generated report
@@ -13,8 +13,8 @@ Application Options:
       --client-key=KEY-FILE      TLS client private key file (PEM encoded data)
       --version                  Display version of the binary
   -v, --verbose                  Verbose output
-  -r, --rate=                    The average upper limit of https operations
+  -r, --rate=                    The average upper limit of https operations per second
-                                 per second
+  -y, --years=YEARS              Number of years to look back from now
 
 Help Options:
   -h, --help                     Show this help message