Merge pull request #267 from csaf-poc/directory-url

Directory url

cmd/csaf_aggregator/mirror.go

@@ -131,6 +131,19 @@ func (w *worker) writeProviderMetadata() error {
 		w.processor.cfg.Domain+"/.well-known/csaf-aggregator/"+w.provider.Name,
 		w.labelsFromSummaries())
 
+	// Fill in directory URLs if needed.
+	if w.provider.writeIndices(w.processor.cfg) {
+		labels := make([]string, 0, len(w.summaries))
+		for label := range w.summaries {
+			labels = append(labels, label)
+		}
+		sort.Strings(labels)
+		prefix := w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" + w.provider.Name + "/"
+		for _, label := range labels {
+			pm.AddDirectoryDistribution(prefix + label)
+		}
+	}
+
 	// Figure out the role
 	var role csaf.MetadataRole
 

cmd/csaf_checker/processor.go

@@ -395,12 +395,21 @@ func (p *processor) integrity(
 
 	var data bytes.Buffer
 
+	makeAbs := func(u *url.URL) *url.URL {
+		if u.IsAbs() {
+			return u
+		}
+		return util.JoinURLPath(b, u.String())
+	}
+
 	for _, f := range files {
 		fp, err := url.Parse(f.URL())
 		if err != nil {
 			lg(ErrorType, "Bad URL %s: %v", f, err)
 			continue
 		}
+		fp = makeAbs(fp)
+
 		u := b.ResolveReference(fp).String()
 		if p.markChecked(u, mask) {
 			continue
@@ -492,6 +501,7 @@ func (p *processor) integrity(
 				lg(ErrorType, "Bad URL %s: %v", x.url(), err)
 				continue
 			}
+			hu = makeAbs(hu)
 			hashFile := b.ResolveReference(hu).String()
 			p.checkTLS(hashFile)
 			if res, err = client.Get(hashFile); err != nil {
@@ -527,6 +537,7 @@ func (p *processor) integrity(
 			lg(ErrorType, "Bad URL %s: %v", f.SignURL(), err)
 			continue
 		}
+		su = makeAbs(su)
 		sigFile := b.ResolveReference(su).String()
 		p.checkTLS(sigFile)
 
@@ -822,9 +833,10 @@ func (p *processor) checkChanges(base string, mask whereType) error {
 			if p.ageAccept != nil && !p.ageAccept(t) {
 				continue
 			}
+			path := r[pathColumn]
 			times, files =
 				append(times, t),
-				append(files, csaf.PlainAdvisoryFile(r[pathColumn]))
+				append(files, csaf.PlainAdvisoryFile(path))
 		}
 		return times, files, nil
 	}()
@@ -877,6 +889,16 @@ func (p *processor) processROLIEFeeds(domain string, feeds [][]csaf.Feed) error
 	return nil
 }
 
+// empty checks if list of strings contains at least one none empty string.
+func empty(arr []string) bool {
+	for _, s := range arr {
+		if s != "" {
+			return false
+		}
+	}
+	return true
+}
+
 func (p *processor) checkCSAFs(domain string) error {
 	// Check for ROLIE
 	rolie, err := p.expr.Eval("$.distributions[*].rolie.feeds", p.pmd)
@@ -898,22 +920,46 @@ func (p *processor) checkCSAFs(domain string) error {
 		}
 	}
 
-	// No rolie feeds
+	// No rolie feeds -> try directory_urls.
-	pmdURL, err := url.Parse(p.pmdURL)
+	directoryURLs, err := p.expr.Eval(
+		"$.distributions[*].directory_url", p.pmd)
+
+	var dirURLs []string
+
 	if err != nil {
-		return err
+		p.badProviderMetadata.warn("extracting directory URLs failed: %v.", err)
-	}
+	} else {
-	base, err := util.BaseURL(pmdURL)
+		var ok bool
-	if err != nil {
+		dirURLs, ok = util.AsStrings(directoryURLs)
-		return err
+		if !ok {
+			p.badProviderMetadata.warn("directory URLs are not strings.")
+		}
 	}
 
-	if err := p.checkIndex(base, indexMask); err != nil && err != errContinue {
+	// Not found -> fall back to PMD url
-		return err
+	if empty(dirURLs) {
+		pmdURL, err := url.Parse(p.pmdURL)
+		if err != nil {
+			return err
+		}
+		baseURL, err := util.BaseURL(pmdURL)
+		if err != nil {
+			return err
+		}
+		dirURLs = []string{baseURL}
 	}
 
-	if err := p.checkChanges(base, changesMask); err != nil && err != errContinue {
+	for _, base := range dirURLs {
-		return err
+		if base == "" {
+			continue
+		}
+		if err := p.checkIndex(base, indexMask); err != nil && err != errContinue {
+			return err
+		}
+
+		if err := p.checkChanges(base, changesMask); err != nil && err != errContinue {
+			return err
+		}
 	}
 
 	return nil

cmd/csaf_provider/create.go

@@ -298,6 +298,17 @@ func createProviderMetadata(c *config, wellknownCSAF string) error {
 	pm := csaf.NewProviderMetadataDomain(c.CanonicalURLPrefix, c.modelTLPs())
 	c.ProviderMetaData.apply(pm)
 
+	// We have directory based distributions.
+	if c.WriteIndices {
+		// Every TLP as a distribution?
+		for _, t := range c.TLPs {
+			if t != tlpCSAF {
+				pm.AddDirectoryDistribution(
+					c.CanonicalURLPrefix + "/.well-known/csaf/" + string(t))
+			}
+		}
+	}
+
 	key, err := loadCryptoKeyFromFile(c.OpenPGPPublicKey)
 	if err != nil {
 		return fmt.Errorf("cannot load public key: %v", err)

csaf/advisories.go

@@ -96,6 +96,16 @@ func NewAdvisoryFileProcessor(
 	}
 }
 
+// empty checks if list of strings contains at least one none empty string.
+func empty(arr []string) bool {
+	for _, s := range arr {
+		if s != "" {
+			return false
+		}
+	}
+	return true
+}
+
 // Process extracts the adivisory filenames and passes them with
 // the corresponding label to fn.
 func (afp *AdvisoryFileProcessor) Process(
@@ -133,13 +143,44 @@ func (afp *AdvisoryFileProcessor) Process(
 		}
 	} else {
 		// No rolie feeds -> try to load files from index.txt
-		files, err := afp.loadIndex(lg)
+
+		directoryURLs, err := afp.expr.Eval(
+			"$.distributions[*].directory_url", afp.doc)
+
+		var dirURLs []string
+
 		if err != nil {
-			return err
+			lg("extracting directory URLs failed: %v\n", err)
+		} else {
+			var ok bool
+			dirURLs, ok = util.AsStrings(directoryURLs)
+			if !ok {
+				lg("directory_urls are not strings.\n")
+			}
 		}
-		// XXX: Is treating as white okay? better look into the advisories?
+
-		if err := fn(TLPLabelWhite, files); err != nil {
+		// Not found -> fall back to PMD url
-			return err
+		if empty(dirURLs) {
+			baseURL, err := util.BaseURL(afp.base)
+			if err != nil {
+				return err
+			}
+			dirURLs = []string{baseURL}
+		}
+
+		for _, base := range dirURLs {
+			if base == "" {
+				continue
+			}
+
+			files, err := afp.loadIndex(base, lg)
+			if err != nil {
+				return err
+			}
+			// XXX: Is treating as white okay? better look into the advisories?
+			if err := fn(TLPLabelWhite, files); err != nil {
+				return err
+			}
 		}
 	} // TODO: else scan directories?
 	return nil
@@ -148,12 +189,10 @@ func (afp *AdvisoryFileProcessor) Process(
 // loadIndex loads baseURL/index.txt and returns a list of files
 // prefixed by baseURL/.
 func (afp *AdvisoryFileProcessor) loadIndex(
+	baseURL string,
 	lg func(string, ...interface{}),
 ) ([]AdvisoryFile, error) {
-	baseURL, err := util.BaseURL(afp.base)
+
-	if err != nil {
-		return nil, err
-	}
 	base, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err

csaf/models.go

@@ -466,6 +466,18 @@ func (pmd *ProviderMetadata) Defaults() {
 	}
 }
 
+// AddDirectoryDistribution adds a directory based distribution
+// with a given url to the provider metadata.
+func (pmd *ProviderMetadata) AddDirectoryDistribution(url string) {
+	// Avoid duplicates.
+	for i := range pmd.Distributions {
+		if pmd.Distributions[i].DirectoryURL == url {
+			return
+		}
+	}
+	pmd.Distributions = append(pmd.Distributions, Distribution{DirectoryURL: url})
+}
+
 // Validate checks if the feed is valid.
 // Returns an error if the validation fails otherwise nil.
 func (f *Feed) Validate() error {

util/json.go

@@ -166,3 +166,19 @@ func (pe *PathEval) Strings(
 	}
 	return results, nil
 }
+
+// AsStrings transforms an []interface{string, string,... }
+// to a []string.
+func AsStrings(x interface{}) ([]string, bool) {
+	strs, ok := x.([]interface{})
+	if !ok {
+		return nil, false
+	}
+	res := make([]string, 0, len(strs))
+	for _, y := range strs {
+		if s, ok := y.(string); ok {
+			res = append(res, s)
+		}
+	}
+	return res, true
+}