From 594e6b4b0df97e4c406cb5b88d3ee4f968a623e8 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 22 Jun 2023 14:28:45 +0200 Subject: [PATCH] Try to minimize redundant downloads. --- cmd/csaf_checker/main.go | 8 ++++ cmd/csaf_checker/processor.go | 31 ++++++++++--- cmd/csaf_checker/roliecheck.go | 80 ++++++++++++++++++++-------------- 3 files changed, 80 insertions(+), 39 deletions(-) diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 9fd0fa0..eafa9c8 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -74,6 +74,14 @@ func (o *options) prepare() error { return nil } +// protectedAccess returns true if we have client certificates or +// extra http headers configured. +// This may be a wrong assumption, because the certs are not checked +// for their domain and custom headers may have other purposes. +func (o *options) protectedAccess() bool { + return len(o.clientCerts) > 0 || len(o.ExtraHeader) > 0 +} + // writeJSON writes the JSON encoding of the given report to the given stream. // It returns nil, otherwise an error. func writeJSON(report *Report, w io.WriteCloser) error { diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index c883fd2..9b26093 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -407,12 +407,8 @@ func (p *processor) checkRedirect(r *http.Request, via []*http.Request) error { return nil } -func (p *processor) httpClient() util.Client { - - if p.client != nil { - return p.client - } - +// fullClient returns a fully configure HTTP client. +func (p *processor) fullClient() util.Client { hClient := http.Client{} hClient.CheckRedirect = p.checkRedirect @@ -452,8 +448,29 @@ func (p *processor) httpClient() util.Client { Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1), } } + return client +} - p.client = client +// basicClient returns a http Client w/o certs and headers. +func (p *processor) basicClient() *http.Client { + if p.opts.Insecure { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + return &http.Client{Transport: tr} + } + return &http.Client{} +} + +// httpClient returns a cached HTTP client to be used to +// download remote ressources. +func (p *processor) httpClient() util.Client { + + if p.client != nil { + return p.client + } + + p.client = p.fullClient() return p.client } diff --git a/cmd/csaf_checker/roliecheck.go b/cmd/csaf_checker/roliecheck.go index cbcafcc..009af34 100644 --- a/cmd/csaf_checker/roliecheck.go +++ b/cmd/csaf_checker/roliecheck.go @@ -9,7 +9,6 @@ package main import ( - "crypto/tls" "net/http" "net/url" "sort" @@ -25,8 +24,8 @@ type rolieLabelChecker struct { feedURL string feedLabel csaf.TLPLabel - advisories map[csaf.TLPLabel]util.Set[string] - basicClient *http.Client + advisories map[csaf.TLPLabel]util.Set[string] + openClient util.Client } // tlpLevel returns an inclusion order of TLP colors. @@ -55,17 +54,6 @@ func tlpLabel(label *csaf.TLPLabel) csaf.TLPLabel { return csaf.TLPLabelUnlabeled } -// createBasicClient creates and returns a http Client -func (p *processor) createBasicClient() *http.Client { - if p.opts.Insecure { - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - return &http.Client{Transport: tr} - } - return &http.Client{} -} - // check tests if in advisory is in the right TLP color of the // currently tested feed. func (ca *rolieLabelChecker) check( @@ -109,23 +97,38 @@ func (ca *rolieLabelChecker) check( advisory, advisoryLabel, ca.feedURL, ca.feedLabel) } - res, err := ca.basicClient.Get(advisory) - switch { - case advisoryRank == 1: - p.badWhitePermissions.use() - if err != nil { - p.badWhitePermissions.error("Unexpected Error %v when trying to fetch: %s", err, advisory) - } else if res.StatusCode == http.StatusForbidden { - // TODO: Differentiate between error and warning based on whether the advisory appears in a not access protected location as well. - p.badWhitePermissions.warn("Advisory %s of TLP level WHITE is access protected.", advisory) - } - case advisoryRank > 2: - p.badAmberRedPermissions.use() - if err != nil { - p.badAmberRedPermissions.error("Unexpected Error %v when trying to fetch: %s", err, advisory) - } else if res.StatusCode == http.StatusOK { - p.badAmberRedPermissions.error("Advisory %s of TLP level %v is not properly access protected.", advisory, advisoryLabel) + // If we have an open client then the actual data was downloaded + // through an authorizing client. + if ca.openClient != nil { + switch { + // If we are checking WHITE and we have a test client + // and we get a status forbidden then the access is not open. + case ca.feedLabel == csaf.TLPLabelWhite: + p.badWhitePermissions.use() + res, err := ca.openClient.Get(advisory) + if err != nil { + p.badWhitePermissions.error( + "Unexpected Error %v when trying to fetch: %s", err, advisory) + } else if res.StatusCode == http.StatusForbidden { + p.badWhitePermissions.error( + "Advisory %s of TLP level WHITE is access protected.", advisory) + } + // If we are checking AMBER or above we need to download + // the data again with the open client. + // If this does not result in status forbidden the + // server may be wrongly configured. + case ca.feedLabel >= csaf.TLPLabelAmber: + p.badAmberRedPermissions.use() + res, err := ca.openClient.Get(advisory) + if err != nil { + p.badAmberRedPermissions.error( + "Unexpected Error %v when trying to fetch: %s", err, advisory) + } else if res.StatusCode == http.StatusOK { + p.badAmberRedPermissions.error( + "Advisory %s of TLP level %v is not properly access protected.", advisory, advisoryLabel) + + } } } } @@ -168,9 +171,9 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error { advisories[feed] = advs } } + p.labelChecker = &rolieLabelChecker{ - advisories: map[csaf.TLPLabel]util.Set[string]{}, - basicClient: p.createBasicClient(), + advisories: map[csaf.TLPLabel]util.Set[string]{}, } // Phase 2: check for integrity. @@ -208,6 +211,19 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error { p.labelChecker.feedURL = feedURL.String() p.labelChecker.feedLabel = label + // If we are using an authorizing client + // we need a an open client to check + // WHITE, AMBER and RED feeds. + var openClient util.Client + if (label == csaf.TLPLabelWhite || label >= csaf.TLPLabelAmber) && + p.opts.protectedAccess() { + openClient = p.basicClient() + } + p.labelChecker.openClient = openClient + + // TODO: Issue a warning if we want check AMBER+ without an + // authorizing client. + if err := p.integrity(files, feedBase, rolieMask, p.badProviderMetadata.add); err != nil { if err != errContinue { return err