From 55540a32e0282bcfeee57c04add56bece4e9ad90 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 8 Jun 2023 18:16:56 +0200 Subject: [PATCH] Simplified requirement 15 (#369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add badROLIEfeed as Topic Message * Use badROLIEfeed to guarantee existant TLP labels White, Green or unlabeled. (Test not implemented) * syntax * Formatting * Add Tlp check, completion struct * Add mismatch to completion, add function checkCompletion to fill mismatch and also give an error if invalid tlp levels have been used * formatting * Add function to remove incomplete csaf feeds from list of complete csaf feeds for a given tlp level * Add checkSummary function that checks whether a given feed would qualify as summary feed between all currently checked feeds * Add completed check of tlp levels * Add checks for correct hashes and signatures in ROLIE feed * formatting * Add rolieFeedReporter functionality * fix typo * Add todo, add return values to functions * Switch error, ... return value so error returns last * Fix typo * Remove hash/sig checks that don't work, improve ROLIE message * Add handling for advisories without tlp level * Formatting * Clean up rolie checks. * Started with simplifying rolie checking * Every ROLIE with data should have a summary. * Clean up ROLIE feed label checker. * if no TLP level can be extracted, return Unlabeled, not WHITE * Add handling of advisories whose tlp exists, but has no label * Also check TLP Red for completeness * Only remove advisory from remain when it has exactly the right tlp color. * Fix import in new rolie feed checker. * Update comment to reflect current functionality * Accept advisory of lesser tlp color in feed as completing. * Collect advisory labels from advisories. * Clarify that if no summary feed was found, it may exist but be either not listed or not accessible. * Do not clone advisory lookup before. * Move rolie check code to respective file. --------- Co-authored-by: JanHoefelmeyer Co-authored-by: JanHoefelmeyer --- cmd/csaf_checker/processor.go | 315 +++++++++++++++++---------------- cmd/csaf_checker/reporters.go | 13 +- cmd/csaf_checker/roliecheck.go | 265 +++++++++++++++++++++++++++ 3 files changed, 434 insertions(+), 159 deletions(-) create mode 100644 cmd/csaf_checker/roliecheck.go diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index d4e2751..df0cb89 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -52,6 +52,7 @@ type processor struct { pmd256 []byte pmd any keys *crypto.KeyRing + labelChecker *rolieLabelChecker invalidAdvisories topicMessages badFilenames topicMessages @@ -66,6 +67,7 @@ type processor struct { badWellknownMetadata topicMessages badDNSPath topicMessages badDirListings topicMessages + badROLIEfeed topicMessages expr *util.PathEval } @@ -218,6 +220,8 @@ func (p *processor) clean() { p.badWellknownMetadata.reset() p.badDNSPath.reset() p.badDirListings.reset() + p.badROLIEfeed.reset() + p.labelChecker = nil } // run calls checkDomain function for each domain in the given "domains" parameter. @@ -412,6 +416,129 @@ func (p *processor) httpClient() util.Client { return p.client } +// rolieFeedEntries loads the references to the advisory files for a given feed. +func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) { + + client := p.httpClient() + res, err := client.Get(feed) + p.badDirListings.use() + if err != nil { + p.badProviderMetadata.error("Cannot fetch feed %s: %v", feed, err) + return nil, errContinue + } + if res.StatusCode != http.StatusOK { + p.badProviderMetadata.warn("Fetching %s failed. Status code %d (%s)", + feed, res.StatusCode, res.Status) + return nil, errContinue + } + + rfeed, rolieDoc, err := func() (*csaf.ROLIEFeed, any, error) { + defer res.Body.Close() + all, err := io.ReadAll(res.Body) + if err != nil { + return nil, nil, err + } + rfeed, err := csaf.LoadROLIEFeed(bytes.NewReader(all)) + if err != nil { + return nil, nil, fmt.Errorf("%s: %v", feed, err) + } + var rolieDoc any + err = json.NewDecoder(bytes.NewReader(all)).Decode(&rolieDoc) + return rfeed, rolieDoc, err + + }() + if err != nil { + p.badProviderMetadata.error("Loading ROLIE feed failed: %v.", err) + return nil, errContinue + } + errors, err := csaf.ValidateROLIE(rolieDoc) + if err != nil { + return nil, err + } + if len(errors) > 0 { + p.badProviderMetadata.error("%s: Validating against JSON schema failed:", feed) + for _, msg := range errors { + p.badProviderMetadata.error(strings.ReplaceAll(msg, `%`, `%%`)) + } + } + + // Extract the CSAF files from feed. + var files []csaf.AdvisoryFile + + rfeed.Entries(func(entry *csaf.Entry) { + + // Filter if we have date checking. + if p.ageAccept != nil { + if pub := time.Time(entry.Published); !pub.IsZero() && !p.ageAccept(pub) { + return + } + } + + var url, sha256, sha512, sign string + for i := range entry.Link { + link := &entry.Link[i] + lower := strings.ToLower(link.HRef) + switch link.Rel { + case "self": + if !strings.HasSuffix(lower, ".json") { + p.badProviderMetadata.warn( + `ROLIE feed entry link %s in %s with "rel": "self" has unexpected file extension.`, + link.HRef, feed) + } + url = link.HRef + case "signature": + if !strings.HasSuffix(lower, ".asc") { + p.badProviderMetadata.warn( + `ROLIE feed entry link %s in %s with "rel": "signature" has unexpected file extension.`, + link.HRef, feed) + } + sign = link.HRef + case "hash": + switch { + case strings.HasSuffix(lower, "sha256"): + sha256 = link.HRef + case strings.HasSuffix(lower, "sha512"): + sha512 = link.HRef + default: + p.badProviderMetadata.warn( + `ROLIE feed entry link %s in %s with "rel": "hash" has unsupported file extension.`, + link.HRef, feed) + } + } + } + + if url == "" { + p.badProviderMetadata.warn( + `ROLIE feed %s contains entry link with no "self" URL.`, feed) + return + } + + var file csaf.AdvisoryFile + + if sha256 != "" || sha512 != "" || sign != "" { + file = csaf.HashedAdvisoryFile{url, sha256, sha512, sign} + } else { + file = csaf.PlainAdvisoryFile(url) + } + + files = append(files, file) + }) + + return files, nil +} + +// makeAbsolute returns a function that checks if a given +// URL is absolute or not. If not it returns an +// absolute URL based on a given base URL. +func makeAbsolute(base *url.URL) func(*url.URL) *url.URL { + return func(u *url.URL) *url.URL { + if u.IsAbs() { + return u + } + return base.JoinPath(u.String()) + } +} + var yearFromURL = regexp.MustCompile(`.*/(\d{4})/[^/]+$`) func (p *processor) integrity( @@ -424,17 +551,11 @@ func (p *processor) integrity( if err != nil { return err } + makeAbs := makeAbsolute(b) client := p.httpClient() var data bytes.Buffer - makeAbs := func(u *url.URL) *url.URL { - if u.IsAbs() { - return u - } - return b.JoinPath(u.String()) - } - for _, f := range files { fp, err := url.Parse(f.URL()) if err != nil { @@ -456,7 +577,6 @@ func (p *processor) integrity( } var folderYear *int - if m := yearFromURL.FindStringSubmatch(u); m != nil { year, _ := strconv.Atoi(m[1]) // Check if we are in checking time interval. @@ -521,7 +641,6 @@ func (p *processor) integrity( continue } - // Validate against remote validator. if p.validator != nil { if rvr, err := p.validator.Validate(doc); err != nil { @@ -531,6 +650,19 @@ func (p *processor) integrity( } } + // Extract the tlp level of the entry + if tlpa, err := p.expr.Eval( + `$.document.distribution`, doc); err != nil { + p.badROLIEfeed.error( + "Extracting 'tlp level' from %s failed: %v", u, err) + } else { + tlpe := extractTLP(tlpa) + // check if current feed has correct or all of their tlp levels entries. + if p.labelChecker != nil { + p.labelChecker.check(p, tlpe, u) + } + } + // Check if file is in the right folder. p.badFolders.use() @@ -567,6 +699,7 @@ func (p *processor) integrity( } hu = makeAbs(hu) hashFile := b.ResolveReference(hu).String() + p.checkTLS(hashFile) if res, err = client.Get(hashFile); err != nil { p.badIntegrities.error("Fetching %s failed: %v.", hashFile, err) @@ -594,7 +727,6 @@ func (p *processor) integrity( x.ext, u, hashFile) } } - // Check signature su, err := url.Parse(f.SignURL()) if err != nil { @@ -639,133 +771,23 @@ func (p *processor) integrity( } } } + return nil } -func (p *processor) processROLIEFeed(feed string) error { - client := p.httpClient() - res, err := client.Get(feed) - p.badDirListings.use() - if err != nil { - p.badProviderMetadata.error("Cannot fetch feed %s: %v", feed, err) - return errContinue - } - if res.StatusCode != http.StatusOK { - p.badProviderMetadata.warn("Fetching %s failed. Status code %d (%s)", - feed, res.StatusCode, res.Status) - return errContinue - } - - rfeed, rolieDoc, err := func() (*csaf.ROLIEFeed, any, error) { - defer res.Body.Close() - all, err := io.ReadAll(res.Body) - if err != nil { - return nil, nil, err - } - rfeed, err := csaf.LoadROLIEFeed(bytes.NewReader(all)) - if err != nil { - return nil, nil, fmt.Errorf("%s: %v", feed, err) - } - var rolieDoc any - err = json.NewDecoder(bytes.NewReader(all)).Decode(&rolieDoc) - return rfeed, rolieDoc, err - - }() - if err != nil { - p.badProviderMetadata.error("Loading ROLIE feed failed: %v.", err) - return errContinue - } - errors, err := csaf.ValidateROLIE(rolieDoc) - if err != nil { - return err - } - if len(errors) > 0 { - p.badProviderMetadata.error("%s: Validating against JSON schema failed:", feed) - for _, msg := range errors { - p.badProviderMetadata.error(strings.ReplaceAll(msg, `%`, `%%`)) - } - } - - feedURL, err := url.Parse(feed) - if err != nil { - p.badProviderMetadata.error("Bad base path: %v", err) - return errContinue - } - - base, err := util.BaseURL(feedURL) - if err != nil { - p.badProviderMetadata.error("Bad base path: %v", err) - return errContinue - } - - // Extract the CSAF files from feed. - var files []csaf.AdvisoryFile - - rfeed.Entries(func(entry *csaf.Entry) { - - // Filter if we have date checking. - if p.ageAccept != nil { - if pub := time.Time(entry.Published); !pub.IsZero() && !p.ageAccept(pub) { - return - } - } - - var url, sha256, sha512, sign string - - for i := range entry.Link { - link := &entry.Link[i] - lower := strings.ToLower(link.HRef) - switch link.Rel { - case "self": - if !strings.HasSuffix(lower, ".json") { - p.badProviderMetadata.warn( - `ROLIE feed entry link %s in %s with "rel": "self" has unexpected file extension.`, - link.HRef, feed) - } - url = link.HRef - case "signature": - if !strings.HasSuffix(lower, ".asc") { - p.badProviderMetadata.warn( - `ROLIE feed entry link %s in %s with "rel": "signature" has unexpected file extension.`, - link.HRef, feed) - } - sign = link.HRef - case "hash": - switch { - case strings.HasSuffix(lower, "sha256"): - sha256 = link.HRef - case strings.HasSuffix(lower, "sha512"): - sha512 = link.HRef - default: - p.badProviderMetadata.warn( - `ROLIE feed entry link %s in %s with "rel": "hash" has unsupported file extension.`, - link.HRef, feed) +// extractTLP tries to extract a valid TLP label from an advisory +// Returns "UNLABELED" if it does not exist, the label otherwise +func extractTLP(tlpa any) csaf.TLPLabel { + if distribution, ok := tlpa.(map[string]any); ok { + if tlp, ok := distribution["tlp"]; ok { + if label, ok := tlp.(map[string]any); ok { + if labelstring, ok := label["label"].(string); ok { + return csaf.TLPLabel(labelstring) } } } - - if url == "" { - p.badProviderMetadata.warn( - `ROLIE feed %s contains entry link with no "self" URL.`, feed) - return - } - - var file csaf.AdvisoryFile - - if sha256 != "" || sha512 != "" || sign != "" { - file = csaf.HashedAdvisoryFile{url, sha256, sha512, sign} - } else { - file = csaf.PlainAdvisoryFile(url) - } - - files = append(files, file) - }) - if err := p.integrity(files, base, rolieMask, p.badProviderMetadata.add); err != nil && - err != errContinue { - return err } - - return nil + return csaf.TLPLabelUnlabeled } // checkIndex fetches the "index.txt" and calls "checkTLS" method for HTTPS checks. @@ -824,6 +846,9 @@ func (p *processor) checkIndex(base string, mask whereType) error { p.badIntegrities.warn("index.txt contains no URLs") } + // Block rolie checks. + p.labelChecker = nil + return p.integrity(files, base, mask, p.badIndices.add) } @@ -916,36 +941,12 @@ func (p *processor) checkChanges(base string, mask whereType) error { p.badChanges.error("%s is not sorted in descending order", changes) } + // Block rolie checks. + p.labelChecker = nil + return p.integrity(files, base, mask, p.badChanges.add) } -func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error { - - base, err := url.Parse(p.pmdURL) - if err != nil { - return err - } - for _, fs := range feeds { - for i := range fs { - feed := &fs[i] - if feed.URL == nil { - continue - } - up, err := url.Parse(string(*feed.URL)) - if err != nil { - p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err) - continue - } - feedURL := base.ResolveReference(up).String() - p.checkTLS(feedURL) - if err := p.processROLIEFeed(feedURL); err != nil && err != errContinue { - return err - } - } - } - return nil -} - // empty checks if list of strings contains at least one none empty string. func empty(arr []string) bool { for _, s := range arr { diff --git a/cmd/csaf_checker/reporters.go b/cmd/csaf_checker/reporters.go index 511c2d8..d511cb6 100644 --- a/cmd/csaf_checker/reporters.go +++ b/cmd/csaf_checker/reporters.go @@ -364,8 +364,17 @@ func (r *directoryListingsReporter) report(p *processor, domain *Domain) { // given TLP level and whether any of the TLP levels // TLP:WHITE, TLP:GREEN or unlabeled exists and sets the "message" field value // of the "Requirement" struct as a result of that. -func (r *rolieFeedReporter) report(_ *processor, _ *Domain) { - // TODO +func (r *rolieFeedReporter) report(p *processor, domain *Domain) { + req := r.requirement(domain) + if !p.badROLIEfeed.used() { + req.message(InfoType, "No checks on the validity of ROLIE feeds performed.") + return + } + if len(p.badROLIEfeed) == 0 { + req.message(InfoType, "All checked ROLIE feeds validated fine.") + return + } + req.Messages = p.badROLIEfeed } // report tests whether a ROLIE service document is used and if so, diff --git a/cmd/csaf_checker/roliecheck.go b/cmd/csaf_checker/roliecheck.go new file mode 100644 index 0000000..9dcdf11 --- /dev/null +++ b/cmd/csaf_checker/roliecheck.go @@ -0,0 +1,265 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +package main + +import ( + "net/url" + + "github.com/csaf-poc/csaf_distribution/v2/csaf" + "github.com/csaf-poc/csaf_distribution/v2/util" +) + +// rolieLabelChecker helps to check id advisories in ROLIE feeds +// are in there right TLP color. +type rolieLabelChecker struct { + feedURL string + feedLabel csaf.TLPLabel + + advisories map[csaf.TLPLabel]map[string]struct{} +} + +// tlpLevel returns an inclusion order of TLP colors. +func tlpLevel(label csaf.TLPLabel) int { + switch label { + case csaf.TLPLabelWhite: + return 1 + case csaf.TLPLabelGreen: + return 2 + case csaf.TLPLabelAmber: + return 3 + case csaf.TLPLabelRed: + return 4 + default: + return 0 + } +} + +// tlpLabel returns the value of a none-nil pointer +// to a TLPLabel. If pointer is nil unlabeled is returned. +func tlpLabel(label *csaf.TLPLabel) csaf.TLPLabel { + if label != nil { + return *label + } + return csaf.TLPLabelUnlabeled +} + +// check tests if in advisory is in the right TLP color of the +// currently tested feed. +func (ca *rolieLabelChecker) check( + p *processor, + advisoryLabel csaf.TLPLabel, + advisory string, +) { + // Assign int to tlp levels for easy comparison + var ( + advisoryRank = tlpLevel(advisoryLabel) + feedRank = tlpLevel(ca.feedLabel) + ) + + // Associate advisory label to urls. + advs := ca.advisories[advisoryLabel] + if advs == nil { + advs = make(map[string]struct{}) + ca.advisories[advisoryLabel] = advs + } + advs[advisory] = struct{}{} + + // If entry shows up in feed of higher tlp level, + // give out info or warning + switch { + case advisoryRank < feedRank: + if advisoryRank == 0 { // All kinds of 'UNLABELED' + p.badROLIEfeed.info( + "Found unlabeled advisory %q in feed %q.", + advisory, ca.feedURL) + } else { + p.badROLIEfeed.warn( + "Found advisory %q labled TLP:%s in feed %q (TLP:%s).", + advisory, advisoryLabel, + ca.feedURL, ca.feedLabel) + } + + case advisoryRank > feedRank: + // Must not happen, give error + p.badROLIEfeed.error( + "%s of TLP level %s must not be listed in feed %s of TLP level %s", + advisory, advisoryLabel, ca.feedURL, ca.feedLabel) + } +} + +// processROLIEFeeds goes through all ROLIE feeds and checks there +// integriry and completeness. +func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error { + + base, err := url.Parse(p.pmdURL) + if err != nil { + return err + } + p.badROLIEfeed.use() + + advisories := map[*csaf.Feed][]csaf.AdvisoryFile{} + + // Phase 1: load all advisories urls. + for _, fs := range feeds { + for i := range fs { + feed := &fs[i] + if feed.URL == nil { + continue + } + up, err := url.Parse(string(*feed.URL)) + if err != nil { + p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err) + continue + } + feedBase := base.ResolveReference(up) + feedURL := feedBase.String() + p.checkTLS(feedURL) + + advs, err := p.rolieFeedEntries(feedURL) + if err != nil { + if err != errContinue { + return err + } + continue + } + advisories[feed] = advs + } + } + + // Phase 2: check for integrity. + for _, fs := range feeds { + for i := range fs { + feed := &fs[i] + if feed.URL == nil { + continue + } + files := advisories[feed] + if files == nil { + continue + } + + up, err := url.Parse(string(*feed.URL)) + if err != nil { + p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err) + continue + } + + feedURL := base.ResolveReference(up) + feedBase, err := util.BaseURL(feedURL) + if err != nil { + p.badProviderMetadata.error("Bad base path: %v", err) + continue + } + + label := tlpLabel(feed.TLPLabel) + + p.labelChecker = &rolieLabelChecker{ + feedURL: feedURL.String(), + feedLabel: label, + advisories: map[csaf.TLPLabel]map[string]struct{}{}, + } + + if err := p.integrity(files, feedBase, rolieMask, p.badProviderMetadata.add); err != nil { + if err != errContinue { + return err + } + } + } + } + + // Phase 3: Check for completeness. + + hasSummary := map[csaf.TLPLabel]struct{}{} + + var ( + hasUnlabeled = false + hasWhite = false + hasGreen = false + ) + + for _, fs := range feeds { + for i := range fs { + feed := &fs[i] + if feed.URL == nil { + continue + } + files := advisories[feed] + if files == nil { + continue + } + + up, err := url.Parse(string(*feed.URL)) + if err != nil { + p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err) + continue + } + + feedBase := base.ResolveReference(up) + makeAbs := makeAbsolute(feedBase) + label := tlpLabel(feed.TLPLabel) + + switch label { + case csaf.TLPLabelUnlabeled: + hasUnlabeled = true + case csaf.TLPLabelWhite: + hasWhite = true + case csaf.TLPLabelGreen: + hasGreen = true + } + + reference := p.labelChecker.advisories[label] + advisories := make(map[string]struct{}, len(reference)) + + for _, adv := range files { + u, err := url.Parse(adv.URL()) + if err != nil { + p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err) + continue + } + advisories[makeAbs(u).String()] = struct{}{} + } + if containsAllKeys(reference, advisories) { + hasSummary[label] = struct{}{} + } + } + } + + if !hasWhite && !hasGreen && !hasUnlabeled { + p.badROLIEfeed.error( + "One ROLIE feed with a TLP:WHITE, TLP:GREEN or unlabeled tlp must exist, " + + "but none were found.") + } + + // Every TLP level with data should have at least on summary feed. + for _, label := range []csaf.TLPLabel{ + csaf.TLPLabelUnlabeled, + csaf.TLPLabelWhite, + csaf.TLPLabelGreen, + csaf.TLPLabelAmber, + csaf.TLPLabelRed, + } { + if _, ok := hasSummary[label]; !ok && len(p.labelChecker.advisories[label]) > 0 { + p.badROLIEfeed.warn( + "ROLIE feed for TLP:%s has no accessible listed feed covering all advisories.", + label) + } + } + + return nil +} + +// containsAllKeys returns if m2 contains all keys of m1. +func containsAllKeys[K comparable, V any](m1, m2 map[K]V) bool { + for k := range m1 { + if _, ok := m2[k]; !ok { + return false + } + } + return true +}