From 8af0aeea46cdac923e99ef5733c192d00cd6ca8c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sat, 30 Jul 2022 12:20:21 +0200 Subject: [PATCH 1/5] Write directory_urls in provider if write indices. --- cmd/csaf_provider/create.go | 11 +++++++++++ csaf/models.go | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/cmd/csaf_provider/create.go b/cmd/csaf_provider/create.go index 56e5955..f571ca8 100644 --- a/cmd/csaf_provider/create.go +++ b/cmd/csaf_provider/create.go @@ -298,6 +298,17 @@ func createProviderMetadata(c *config, wellknownCSAF string) error { pm := csaf.NewProviderMetadataDomain(c.CanonicalURLPrefix, c.modelTLPs()) c.ProviderMetaData.apply(pm) + // We have directory based distributions. + if c.WriteIndices { + // Every TLP as a distribution? + for _, t := range c.TLPs { + if t != tlpCSAF { + pm.AddDirectoryDistribution( + c.CanonicalURLPrefix + "/.well-known/csaf/" + string(t)) + } + } + } + key, err := loadCryptoKeyFromFile(c.OpenPGPPublicKey) if err != nil { return fmt.Errorf("cannot load public key: %v", err) diff --git a/csaf/models.go b/csaf/models.go index d9197d5..8935d41 100644 --- a/csaf/models.go +++ b/csaf/models.go @@ -466,6 +466,18 @@ func (pmd *ProviderMetadata) Defaults() { } } +// AddDirectoryDistribution adds a directory based distribution +// with a given url to the provider metadata. +func (pmd *ProviderMetadata) AddDirectoryDistribution(url string) { + // Avoid duplicates. + for i := range pmd.Distributions { + if pmd.Distributions[i].DirectoryURL == url { + return + } + } + pmd.Distributions = append(pmd.Distributions, Distribution{DirectoryURL: url}) +} + // Validate checks if the feed is valid. // Returns an error if the validation fails otherwise nil. func (f *Feed) Validate() error { From dce3d1f4a709145592e5e808d33ab83c3a274a8e Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 1 Aug 2022 06:46:05 +0200 Subject: [PATCH 2/5] load advisories via directory_urls --- cmd/csaf_checker/processor.go | 56 +++++++++++++++++++++++++++------- csaf/advisories.go | 57 +++++++++++++++++++++++++++++------ 2 files changed, 93 insertions(+), 20 deletions(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index e7acf35..49b134b 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -877,6 +877,16 @@ func (p *processor) processROLIEFeeds(domain string, feeds [][]csaf.Feed) error return nil } +// empty checks if list of strings contains at least one none empty string. +func empty(arr []string) bool { + for _, s := range arr { + if s != "" { + return false + } + } + return true +} + func (p *processor) checkCSAFs(domain string) error { // Check for ROLIE rolie, err := p.expr.Eval("$.distributions[*].rolie.feeds", p.pmd) @@ -898,22 +908,46 @@ func (p *processor) checkCSAFs(domain string) error { } } - // No rolie feeds - pmdURL, err := url.Parse(p.pmdURL) + // No rolie feeds -> try directory_urls. + directoryURLs, err := p.expr.Eval( + "$.distributions[*].directory_url", p.pmd) + + var dirURLs []string + if err != nil { - return err - } - base, err := util.BaseURL(pmdURL) - if err != nil { - return err + p.badProviderMetadata.warn("extracting directory URLs failed: %v.", err) + } else { + var ok bool + dirURLs, ok = directoryURLs.([]string) + if !ok { + p.badProviderMetadata.warn("directory URLs are not strings.") + } } - if err := p.checkIndex(base, indexMask); err != nil && err != errContinue { - return err + // Not found -> fall back to PMD url + if empty(dirURLs) { + pmdURL, err := url.Parse(p.pmdURL) + if err != nil { + return err + } + baseURL, err := util.BaseURL(pmdURL) + if err != nil { + return err + } + dirURLs = []string{baseURL} } - if err := p.checkChanges(base, changesMask); err != nil && err != errContinue { - return err + for _, base := range dirURLs { + if base == "" { + continue + } + if err := p.checkIndex(base, indexMask); err != nil && err != errContinue { + return err + } + + if err := p.checkChanges(base, changesMask); err != nil && err != errContinue { + return err + } } return nil diff --git a/csaf/advisories.go b/csaf/advisories.go index 1e0bdbe..84be1d9 100644 --- a/csaf/advisories.go +++ b/csaf/advisories.go @@ -96,6 +96,16 @@ func NewAdvisoryFileProcessor( } } +// empty checks if list of strings contains at least one none empty string. +func empty(arr []string) bool { + for _, s := range arr { + if s != "" { + return false + } + } + return true +} + // Process extracts the adivisory filenames and passes them with // the corresponding label to fn. func (afp *AdvisoryFileProcessor) Process( @@ -133,13 +143,44 @@ func (afp *AdvisoryFileProcessor) Process( } } else { // No rolie feeds -> try to load files from index.txt - files, err := afp.loadIndex(lg) + + directoryURLs, err := afp.expr.Eval( + "$.distributions[*].directory_url", afp.doc) + + var dirURLs []string + if err != nil { - return err + lg("extracting directory URLs failed: %v\n", err) + } else { + var ok bool + dirURLs, ok = directoryURLs.([]string) + if !ok { + lg("directory_urls are not strings.\n") + } } - // XXX: Is treating as white okay? better look into the advisories? - if err := fn(TLPLabelWhite, files); err != nil { - return err + + // Not found -> fall back to PMD url + if empty(dirURLs) { + baseURL, err := util.BaseURL(afp.base) + if err != nil { + return err + } + dirURLs = []string{baseURL} + } + + for _, base := range dirURLs { + if base == "" { + continue + } + + files, err := afp.loadIndex(base, lg) + if err != nil { + return err + } + // XXX: Is treating as white okay? better look into the advisories? + if err := fn(TLPLabelWhite, files); err != nil { + return err + } } } // TODO: else scan directories? return nil @@ -148,12 +189,10 @@ func (afp *AdvisoryFileProcessor) Process( // loadIndex loads baseURL/index.txt and returns a list of files // prefixed by baseURL/. func (afp *AdvisoryFileProcessor) loadIndex( + baseURL string, lg func(string, ...interface{}), ) ([]AdvisoryFile, error) { - baseURL, err := util.BaseURL(afp.base) - if err != nil { - return nil, err - } + base, err := url.Parse(baseURL) if err != nil { return nil, err From 8e13d37756d0de7fdb5832023854149f2772eb98 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 1 Aug 2022 07:09:21 +0200 Subject: [PATCH 3/5] If we do directory distributions write directory_urls --- cmd/csaf_aggregator/mirror.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go index f9534f0..53206cf 100644 --- a/cmd/csaf_aggregator/mirror.go +++ b/cmd/csaf_aggregator/mirror.go @@ -131,6 +131,19 @@ func (w *worker) writeProviderMetadata() error { w.processor.cfg.Domain+"/.well-known/csaf-aggregator/"+w.provider.Name, w.labelsFromSummaries()) + // Fill in directory URLs if needed. + if w.provider.writeIndices(w.processor.cfg) { + labels := make([]string, 0, len(w.summaries)) + for label := range w.summaries { + labels = append(labels, label) + } + sort.Strings(labels) + prefix := w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" + w.provider.Name + "/" + for _, label := range labels { + pm.AddDirectoryDistribution(prefix + label) + } + } + // Figure out the role var role csaf.MetadataRole From 050e225d07453034363144a7c5396ce31f1546bd Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 1 Aug 2022 13:00:10 +0200 Subject: [PATCH 4/5] Fix type assertions from directory_url expression result --- cmd/csaf_checker/processor.go | 2 +- csaf/advisories.go | 2 +- util/json.go | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 49b134b..14cd6de 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -918,7 +918,7 @@ func (p *processor) checkCSAFs(domain string) error { p.badProviderMetadata.warn("extracting directory URLs failed: %v.", err) } else { var ok bool - dirURLs, ok = directoryURLs.([]string) + dirURLs, ok = util.AsStrings(directoryURLs) if !ok { p.badProviderMetadata.warn("directory URLs are not strings.") } diff --git a/csaf/advisories.go b/csaf/advisories.go index 84be1d9..66f713b 100644 --- a/csaf/advisories.go +++ b/csaf/advisories.go @@ -153,7 +153,7 @@ func (afp *AdvisoryFileProcessor) Process( lg("extracting directory URLs failed: %v\n", err) } else { var ok bool - dirURLs, ok = directoryURLs.([]string) + dirURLs, ok = util.AsStrings(directoryURLs) if !ok { lg("directory_urls are not strings.\n") } diff --git a/util/json.go b/util/json.go index bd53f64..6f2288b 100644 --- a/util/json.go +++ b/util/json.go @@ -166,3 +166,19 @@ func (pe *PathEval) Strings( } return results, nil } + +// AsStrings transforms an []interface{string, string,... } +// to a []string. +func AsStrings(x interface{}) ([]string, bool) { + strs, ok := x.([]interface{}) + if !ok { + return nil, false + } + res := make([]string, 0, len(strs)) + for _, y := range strs { + if s, ok := y.(string); ok { + res = append(res, s) + } + } + return res, true +} From 29d7dd122365d665bbb3eec0578860d1eee496d6 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 1 Aug 2022 14:46:57 +0200 Subject: [PATCH 5/5] Make URLs absolute in integrity check --- cmd/csaf_checker/processor.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 0eaf97a..86eb387 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -395,12 +395,21 @@ func (p *processor) integrity( var data bytes.Buffer + makeAbs := func(u *url.URL) *url.URL { + if u.IsAbs() { + return u + } + return util.JoinURLPath(b, u.String()) + } + for _, f := range files { fp, err := url.Parse(f.URL()) if err != nil { lg(ErrorType, "Bad URL %s: %v", f, err) continue } + fp = makeAbs(fp) + u := b.ResolveReference(fp).String() if p.markChecked(u, mask) { continue @@ -492,6 +501,7 @@ func (p *processor) integrity( lg(ErrorType, "Bad URL %s: %v", x.url(), err) continue } + hu = makeAbs(hu) hashFile := b.ResolveReference(hu).String() p.checkTLS(hashFile) if res, err = client.Get(hashFile); err != nil { @@ -527,6 +537,7 @@ func (p *processor) integrity( lg(ErrorType, "Bad URL %s: %v", f.SignURL(), err) continue } + su = makeAbs(su) sigFile := b.ResolveReference(su).String() p.checkTLS(sigFile) @@ -822,9 +833,10 @@ func (p *processor) checkChanges(base string, mask whereType) error { if p.ageAccept != nil && !p.ageAccept(t) { continue } + path := r[pathColumn] times, files = append(times, t), - append(files, csaf.PlainAdvisoryFile(r[pathColumn])) + append(files, csaf.PlainAdvisoryFile(path)) } return times, files, nil }()