From ec0548bef6bea227c86fe9e3c68b5e6bab0b9203 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 17 Jul 2022 12:27:25 +0200 Subject: [PATCH 1/7] introduce quick check bases on folder year --- cmd/csaf_checker/main.go | 1 + cmd/csaf_checker/processor.go | 36 +++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 23757bc..93c33d5 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -34,6 +34,7 @@ type options struct { Version bool `long:"version" description:"Display version of the binary"` Verbose bool `long:"verbose" short:"v" description:"Verbose output"` Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"` + Years *uint `long:"years" short:"y" description:"The years to look back from now" value-name:"YEARS"` } func errCheck(err error) { diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 92d80ba..a1f44d2 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -40,8 +40,9 @@ import ( type topicMessages []Message type processor struct { - opts *options - client util.Client + opts *options + client util.Client + ageAccept func(time.Time) bool redirects map[string][]string noneTLS map[string]struct{} @@ -159,6 +160,17 @@ func newProcessor(opts *options) *processor { opts: opts, alreadyChecked: map[string]whereType{}, expr: util.NewPathEval(), + ageAccept: ageAccept(opts), + } +} + +func ageAccept(opts *options) func(time.Time) bool { + if opts.Years == nil { + return nil + } + good := time.Now().AddDate(-int(*opts.Years), 0, 0) + return func(t time.Time) bool { + return !t.Before(good) } } @@ -354,6 +366,22 @@ func (p *processor) integrity( continue } p.checkTLS(u) + + var folderYear *int + + if m := yearFromURL.FindStringSubmatch(u); m != nil { + year, _ := strconv.Atoi(m[1]) + folderYear = &year + // Check if we are in checking time interval. + if p.ageAccept != nil && !p.ageAccept( + time.Date( + year+1, 1, 1, // Assume 1. jan of next year. + 0, 0, 0, 0, + time.UTC)) { + continue + } + } + res, err := client.Get(u) if err != nil { lg(ErrorType, "Fetching %s failed: %v.", u, err) @@ -402,9 +430,9 @@ func (p *processor) integrity( } else if d, err := time.Parse(time.RFC3339, text); err != nil { p.badFolders.error( "Parsing 'initial_release_date' as RFC3339 failed in %s: %v", u, err) - } else if m := yearFromURL.FindStringSubmatch(u); m == nil { + } else if folderYear == nil { p.badFolders.error("No year folder found in %s", u) - } else if year, _ := strconv.Atoi(m[1]); d.UTC().Year() != year { + } else if d.UTC().Year() != *folderYear { p.badFolders.error("%s should be in folder %d", u, d.UTC().Year()) } From b39553fc9a95f522f06f0999298bab4f2d92ff8a Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 17 Jul 2022 14:35:37 +0200 Subject: [PATCH 2/7] If using folder per year based date checks assume last second of 31 december --- cmd/csaf_checker/processor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index a1f44d2..c158fcc 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -371,15 +371,15 @@ func (p *processor) integrity( if m := yearFromURL.FindStringSubmatch(u); m != nil { year, _ := strconv.Atoi(m[1]) - folderYear = &year // Check if we are in checking time interval. if p.ageAccept != nil && !p.ageAccept( time.Date( - year+1, 1, 1, // Assume 1. jan of next year. - 0, 0, 0, 0, + year, 12, 31, // Assume last day og year. + 23, 59, 59, 0, // 23:59:59 time.UTC)) { continue } + folderYear = &year } res, err := client.Get(u) From 8c53b4068b9e8fbb5c6181d43762cbb28e7e2f9b Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 17 Jul 2022 14:46:28 +0200 Subject: [PATCH 3/7] Filter ROLIE entries if we have a date range --- cmd/csaf_checker/processor.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index c158fcc..38fa64d 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -594,6 +594,13 @@ func (p *processor) processROLIEFeed(feed string) error { rfeed.Entries(func(entry *csaf.Entry) { + // Filter if we have date checking. + if p.ageAccept != nil { + if pub := time.Time(entry.Published); !pub.IsZero() && !p.ageAccept(pub) { + return + } + } + var url, sha256, sha512, sign string for i := range entry.Link { From 7e850f7a2f5e07f145d38824be57d97b2fcfa627 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 17 Jul 2022 14:52:16 +0200 Subject: [PATCH 4/7] Apply date range filtering to changes.csv --- cmd/csaf_checker/processor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 38fa64d..ec88c85 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -755,6 +755,10 @@ func (p *processor) checkChanges(base string, mask whereType) error { if err != nil { return nil, nil, err } + // Apply date range filtering. + if p.ageAccept != nil && !p.ageAccept(t) { + continue + } times, files = append(times, t), append(files, csaf.PlainAdvisoryFile(r[1])) } return times, files, nil From 7f113f1f3022bd551eb227d1909c754dd15566b2 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 17 Jul 2022 15:01:15 +0200 Subject: [PATCH 5/7] Adjusted documentation --- cmd/csaf_checker/main.go | 2 +- docs/csaf_checker.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 93c33d5..d4c8ac5 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -34,7 +34,7 @@ type options struct { Version bool `long:"version" description:"Display version of the binary"` Verbose bool `long:"verbose" short:"v" description:"Verbose output"` Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"` - Years *uint `long:"years" short:"y" description:"The years to look back from now" value-name:"YEARS"` + Years *uint `long:"years" short:"y" description:"Number of years to look back from now" value-name:"YEARS"` } func errCheck(err error) { diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index bcfe8e8..7e5b14e 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -3,7 +3,7 @@ ### Usage ``` - csaf_checker [OPTIONS] +csaf_checker [OPTIONS] Application Options: -o, --output=REPORT-FILE File name of the generated report @@ -13,8 +13,8 @@ Application Options: --client-key=KEY-FILE TLS client private key file (PEM encoded data) --version Display version of the binary -v, --verbose Verbose output - -r, --rate= The average upper limit of https operations - per second + -r, --rate= The average upper limit of https operations per second + -y, --years=YEARS Number of years to look back from now Help Options: -h, --help Show this help message From 332f0b2711572e3dd29521fba03bf675ba57e18d Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 18 Jul 2022 10:27:59 +0200 Subject: [PATCH 6/7] Only report bad directories in listing check only once. --- cmd/csaf_checker/links.go | 12 +++++++++++- cmd/csaf_checker/processor.go | 4 +++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/csaf_checker/links.go b/cmd/csaf_checker/links.go index 02e649a..70c70dd 100644 --- a/cmd/csaf_checker/links.go +++ b/cmd/csaf_checker/links.go @@ -25,7 +25,11 @@ type ( pages map[string]*pageContent ) -func (pgs pages) listed(path string, pro *processor) (bool, error) { +func (pgs pages) listed( + path string, + pro *processor, + badDirs map[string]struct{}, +) (bool, error) { pathURL, err := url.Parse(path) if err != nil { return false, err @@ -50,6 +54,10 @@ func (pgs pages) listed(path string, pro *processor) (bool, error) { return false, err } + if _, ok := badDirs[base]; ok { + return false, errContinue + } + // load page client := pro.httpClient() pro.checkTLS(base) @@ -59,11 +67,13 @@ func (pgs pages) listed(path string, pro *processor) (bool, error) { if err != nil { pro.badDirListings.error("Fetching %s failed: %v", base, err) + badDirs[base] = struct{}{} return false, errContinue } if res.StatusCode != http.StatusOK { pro.badDirListings.error("Fetching %s failed. Status code %d (%s)", base, res.StatusCode, res.Status) + badDirs[base] = struct{}{} return false, errContinue } diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 92d80ba..9010cd9 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -873,8 +873,10 @@ func (p *processor) checkListing(string) error { var unlisted []string + badDirs := map[string]struct{}{} + for f := range p.alreadyChecked { - found, err := pgs.listed(f, p) + found, err := pgs.listed(f, p, badDirs) if err != nil && err != errContinue { return err } From 4c1fdd2289ca984d7b07da7569833a272ab61a3e Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 19 Jul 2022 16:14:56 +0200 Subject: [PATCH 7/7] simplified loading of provider metadata in case of dns fallback. (#240) --- csaf/util.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/csaf/util.go b/csaf/util.go index c5997ac..18d5484 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -223,20 +223,8 @@ func LoadProviderMetadataForDomain( // Last resort: fall back to DNS. dnsURL := "https://csaf.data.security." + domain dnsResult := LoadProviderMetadataFromURL(client, dnsURL) - - if dnsResult == nil { - logging("%s not found.", dnsURL) - } else if len(dnsResult.Messages) > 0 { - for _, msg := range dnsResult.Messages { - logging(msg) - } - } else { - // DNS seems to be okay. - return dnsResult - } - - // We failed all. - return nil + lg(dnsResult, dnsURL) + return dnsResult } // ExtractProviderURL extracts URLs of provider metadata.