mirror of
https://github.com/gocsaf/csaf.git
synced 2025-12-22 05:40:11 +01:00
Add aggregator; improve itest workflow
* Factor JSON evaluation and construction base URLs out of of checker. * Move json path matching to util. * Add csaf_aggregator (as additional command) * Improve itest workflow to checkout the branch where it is running on. resolve #105 resolve #72 Co-authored-by: tschmidtb51 <65305130+tschmidtb51@users.noreply.github.com> Co-authored-by: Bernhard Reiter <bernhard@intevation.de> Co-authored-by: Fadi Abbud <fadi.abbud@intevation.de>
This commit is contained in:
Sascha L. Teichmann
GitHub
parent
9da0589236
commit
8a1ebe0b7a
30 changed files with 2789 additions and 88 deletions
516
cmd/csaf_aggregator/mirror.go
Normal file
516
cmd/csaf_aggregator/mirror.go
Normal file
|
|
@ -0,0 +1,516 @@
|
|||
// This file is Free Software under the MIT License
|
||||
// without warranty, see README.md and LICENSES/MIT.txt for details.
|
||||
//
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
|
||||
// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ProtonMail/gopenpgp/v2/crypto"
|
||||
"github.com/csaf-poc/csaf_distribution/csaf"
|
||||
"github.com/csaf-poc/csaf_distribution/util"
|
||||
)
|
||||
|
||||
func (w *worker) handleROLIE(
|
||||
rolie interface{},
|
||||
process func(*csaf.TLPLabel, []string) error,
|
||||
) error {
|
||||
base, err := url.Parse(w.loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var feeds [][]csaf.Feed
|
||||
if err := util.ReMarshalJSON(&feeds, rolie); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Found %d ROLIE feed(s).\n", len(feeds))
|
||||
|
||||
for _, fs := range feeds {
|
||||
for i := range fs {
|
||||
feed := &fs[i]
|
||||
if feed.URL == nil {
|
||||
continue
|
||||
}
|
||||
up, err := url.Parse(string(*feed.URL))
|
||||
if err != nil {
|
||||
log.Printf("Invalid URL %s in feed: %v.", *feed.URL, err)
|
||||
continue
|
||||
}
|
||||
feedURL := base.ResolveReference(up).String()
|
||||
log.Printf("Feed URL: %s\n", feedURL)
|
||||
|
||||
fb, err := util.BaseURL(feedURL)
|
||||
if err != nil {
|
||||
log.Printf("error: Invalid feed base URL '%s': %v\n", fb, err)
|
||||
continue
|
||||
}
|
||||
feedBaseURL, err := url.Parse(fb)
|
||||
if err != nil {
|
||||
log.Printf("error: Cannot parse feed base URL '%s': %v\n", fb, err)
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := w.client.Get(feedURL)
|
||||
if err != nil {
|
||||
log.Printf("error: Cannot get feed '%s'\n", err)
|
||||
continue
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
log.Printf("error: Fetching %s failed. Status code %d (%s)",
|
||||
feedURL, res.StatusCode, res.Status)
|
||||
continue
|
||||
}
|
||||
rfeed, err := func() (*csaf.ROLIEFeed, error) {
|
||||
defer res.Body.Close()
|
||||
return csaf.LoadROLIEFeed(res.Body)
|
||||
}()
|
||||
if err != nil {
|
||||
log.Printf("Loading ROLIE feed failed: %v.", err)
|
||||
continue
|
||||
}
|
||||
files := resolveURLs(rfeed.Files(), feedBaseURL)
|
||||
if err := process(feed.TLPLabel, files); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// mirrorAllowed checks if mirroring is allowed.
|
||||
func (w *worker) mirrorAllowed() bool {
|
||||
var b bool
|
||||
return w.expr.Extract(
|
||||
`$.mirror_on_CSAF_aggregators`,
|
||||
util.BoolMatcher(&b), false, w.metadataProvider) == nil && b
|
||||
}
|
||||
|
||||
func (w *worker) mirror() (*csaf.AggregatorCSAFProvider, error) {
|
||||
result, err := w.mirrorInternal()
|
||||
if err != nil && w.dir != "" {
|
||||
// If something goes wrong remove the debris.
|
||||
if err := os.RemoveAll(w.dir); err != nil {
|
||||
log.Printf("error: %v\n", err)
|
||||
}
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (w *worker) mirrorInternal() (*csaf.AggregatorCSAFProvider, error) {
|
||||
|
||||
// Check if we are allowed to mirror this domain.
|
||||
if !w.mirrorAllowed() {
|
||||
return nil, fmt.Errorf(
|
||||
"no mirroring of '%s' allowed", w.provider.Name)
|
||||
}
|
||||
|
||||
// Collecting the summaries of the advisories.
|
||||
w.summaries = make(map[string][]summary)
|
||||
|
||||
// Check if we have ROLIE feeds.
|
||||
rolie, err := w.expr.Eval(
|
||||
"$.distributions[*].rolie.feeds", w.metadataProvider)
|
||||
if err != nil {
|
||||
log.Printf("rolie check failed: %v\n", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fs, hasRolie := rolie.([]interface{})
|
||||
hasRolie = hasRolie && len(fs) > 0
|
||||
|
||||
if hasRolie {
|
||||
if err := w.handleROLIE(rolie, w.mirrorFiles); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// No rolie feeds -> try to load files from index.txt
|
||||
baseURL, err := util.BaseURL(w.loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files, err := w.loadIndex(baseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = files
|
||||
// XXX: Is treating as white okay? better look into the advisories?
|
||||
white := csaf.TLPLabel(csaf.TLPLabelWhite)
|
||||
if err := w.mirrorFiles(&white, files); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} // TODO: else scan directories?
|
||||
|
||||
if err := w.writeIndices(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := w.doMirrorTransaction(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := w.writeProviderMetadata(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
acp, err := w.createAggregatorProvider()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add us as a miiror.
|
||||
mirrorURL := csaf.ProviderURL(
|
||||
fmt.Sprintf("%s/.well-known/csaf-aggregator/%s/provider-metadata.json",
|
||||
w.cfg.Domain, w.provider.Name))
|
||||
|
||||
acp.Mirrors = []csaf.ProviderURL{
|
||||
mirrorURL,
|
||||
}
|
||||
|
||||
return acp, err
|
||||
}
|
||||
|
||||
func (w *worker) labelsFromSummaries() []csaf.TLPLabel {
|
||||
labels := make([]csaf.TLPLabel, 0, len(w.summaries))
|
||||
for label := range w.summaries {
|
||||
labels = append(labels, csaf.TLPLabel(label))
|
||||
}
|
||||
sort.Slice(labels, func(i, j int) bool { return labels[i] < labels[j] })
|
||||
return labels
|
||||
}
|
||||
|
||||
// writeProviderMetadata writes a local provider metadata for a mirror.
|
||||
func (w *worker) writeProviderMetadata() error {
|
||||
|
||||
fname := filepath.Join(w.dir, "provider-metadata.json")
|
||||
|
||||
pm := csaf.NewProviderMetadataDomain(
|
||||
w.cfg.Domain,
|
||||
w.labelsFromSummaries())
|
||||
|
||||
// Figure out the role
|
||||
var role csaf.MetadataRole
|
||||
|
||||
if strings.HasPrefix(w.provider.Domain, "https://") {
|
||||
role = csaf.MetadataRolePublisher
|
||||
} else {
|
||||
role = csaf.MetadataRoleProvider
|
||||
}
|
||||
|
||||
pm.Role = &role
|
||||
|
||||
pm.Publisher = new(csaf.Publisher)
|
||||
|
||||
var lastUpdate time.Time
|
||||
|
||||
if err := w.expr.Match([]util.PathEvalMatcher{
|
||||
{Expr: `$.publisher`, Action: util.ReMarshalMatcher(pm.Publisher)},
|
||||
{Expr: `$.last_updated`, Action: util.TimeMatcher(&lastUpdate, time.RFC3339)},
|
||||
{Expr: `$.public_openpgp_keys`, Action: util.ReMarshalMatcher(&pm.PGPKeys)},
|
||||
}, w.metadataProvider); err != nil {
|
||||
// only log the errors
|
||||
log.Printf("extracting data from orignal provider failed: %v\n", err)
|
||||
}
|
||||
|
||||
key, err := w.cfg.cryptoKey()
|
||||
if err != nil {
|
||||
log.Printf("error: %v\n", err)
|
||||
}
|
||||
if key != nil {
|
||||
pm.SetPGP(key.GetFingerprint(), w.cfg.GetOpenPGPURL(key))
|
||||
}
|
||||
|
||||
la := csaf.TimeStamp(lastUpdate)
|
||||
pm.LastUpdated = &la
|
||||
|
||||
return util.WriteToFile(fname, pm)
|
||||
}
|
||||
|
||||
// createAggregatorProvider, der the "metadata" section in the "csaf_providers" of
|
||||
// the aggregator document.
|
||||
func (w *worker) createAggregatorProvider() (*csaf.AggregatorCSAFProvider, error) {
|
||||
const (
|
||||
lastUpdatedExpr = `$.last_updated`
|
||||
publisherExpr = `$.publisher`
|
||||
roleExpr = `$.role`
|
||||
urlExpr = `$.canonical_url`
|
||||
)
|
||||
|
||||
var (
|
||||
lastUpdatedT time.Time
|
||||
pub csaf.Publisher
|
||||
roleS string
|
||||
urlS string
|
||||
)
|
||||
|
||||
if err := w.expr.Match([]util.PathEvalMatcher{
|
||||
{Expr: lastUpdatedExpr, Action: util.TimeMatcher(&lastUpdatedT, time.RFC3339)},
|
||||
{Expr: publisherExpr, Action: util.ReMarshalMatcher(&pub)},
|
||||
{Expr: roleExpr, Action: util.StringMatcher(&roleS)},
|
||||
{Expr: urlExpr, Action: util.StringMatcher(&urlS)},
|
||||
}, w.metadataProvider); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var (
|
||||
lastUpdated = csaf.TimeStamp(lastUpdatedT)
|
||||
role = csaf.MetadataRole(roleS)
|
||||
url = csaf.ProviderURL(urlS)
|
||||
)
|
||||
|
||||
return &csaf.AggregatorCSAFProvider{
|
||||
Metadata: &csaf.AggregatorCSAFProviderMetadata{
|
||||
LastUpdated: &lastUpdated,
|
||||
Publisher: &pub,
|
||||
Role: &role,
|
||||
URL: &url,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// doMirrorTransaction performs an atomic directory swap.
|
||||
func (w *worker) doMirrorTransaction() error {
|
||||
|
||||
webTarget := filepath.Join(
|
||||
w.cfg.Web, ".well-known", "csaf-aggregator", w.provider.Name)
|
||||
|
||||
var oldWeb string
|
||||
|
||||
// Resolve old to be removed later
|
||||
if _, err := os.Stat(webTarget); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
os.RemoveAll(w.dir)
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if oldWeb, err = filepath.EvalSymlinks(webTarget); err != nil {
|
||||
os.RemoveAll(w.dir)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Check if there is a sysmlink already.
|
||||
target := filepath.Join(w.cfg.Folder, w.provider.Name)
|
||||
log.Printf("target: '%s'\n", target)
|
||||
|
||||
exists, err := util.PathExists(target)
|
||||
if err != nil {
|
||||
os.RemoveAll(w.dir)
|
||||
return err
|
||||
}
|
||||
|
||||
if exists {
|
||||
if err := os.RemoveAll(target); err != nil {
|
||||
os.RemoveAll(w.dir)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("sym link: %s -> %s\n", w.dir, target)
|
||||
|
||||
// Create a new symlink
|
||||
if err := os.Symlink(w.dir, target); err != nil {
|
||||
os.RemoveAll(w.dir)
|
||||
return err
|
||||
}
|
||||
|
||||
// Move the symlink
|
||||
log.Printf("Move: %s -> %s\n", target, webTarget)
|
||||
if err := os.Rename(target, webTarget); err != nil {
|
||||
os.RemoveAll(w.dir)
|
||||
return err
|
||||
}
|
||||
|
||||
// Finally remove the old folder.
|
||||
if oldWeb != "" {
|
||||
return os.RemoveAll(oldWeb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadSignature downloads an OpenPGP signature from a given url.
|
||||
func (w *worker) downloadSignature(path string) (string, error) {
|
||||
res, err := w.client.Get(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return "", errNotFound
|
||||
}
|
||||
data, err := func() ([]byte, error) {
|
||||
defer res.Body.Close()
|
||||
return io.ReadAll(res.Body)
|
||||
}()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result := string(data)
|
||||
if _, err := crypto.NewPGPMessageFromArmored(result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// sign signs the given data with the configured key.
|
||||
func (w *worker) sign(data []byte) (string, error) {
|
||||
if w.signRing == nil {
|
||||
key, err := w.cfg.cryptoKey()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if key == nil {
|
||||
return "", nil
|
||||
}
|
||||
if pp := w.cfg.Passphrase; pp != nil {
|
||||
if key, err = key.Unlock([]byte(*pp)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if w.signRing, err = crypto.NewKeyRing(key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
sig, err := w.signRing.SignDetached(crypto.NewPlainMessage(data))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sig.GetArmored()
|
||||
}
|
||||
|
||||
func (w *worker) mirrorFiles(tlpLabel *csaf.TLPLabel, files []string) error {
|
||||
label := "unknown"
|
||||
if tlpLabel != nil {
|
||||
label = strings.ToLower(string(*tlpLabel))
|
||||
}
|
||||
|
||||
summaries := w.summaries[label]
|
||||
|
||||
dir, err := w.createDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var content bytes.Buffer
|
||||
|
||||
yearDirs := make(map[int]string)
|
||||
|
||||
for _, file := range files {
|
||||
u, err := url.Parse(file)
|
||||
if err != nil {
|
||||
log.Printf("error: %s\n", err)
|
||||
continue
|
||||
}
|
||||
filename := util.CleanFileName(filepath.Base(u.Path))
|
||||
|
||||
var advisory interface{}
|
||||
|
||||
s256 := sha256.New()
|
||||
s512 := sha512.New()
|
||||
content.Reset()
|
||||
hasher := io.MultiWriter(s256, s512, &content)
|
||||
|
||||
download := func(r io.Reader) error {
|
||||
tee := io.TeeReader(r, hasher)
|
||||
return json.NewDecoder(tee).Decode(&advisory)
|
||||
}
|
||||
|
||||
if err := downloadJSON(w.client, file, download); err != nil {
|
||||
log.Printf("error: %v\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
errors, err := csaf.ValidateCSAF(advisory)
|
||||
if err != nil {
|
||||
log.Printf("error: %s: %v", file, err)
|
||||
continue
|
||||
}
|
||||
if len(errors) > 0 {
|
||||
log.Printf("CSAF file %s has %d validation errors.",
|
||||
file, len(errors))
|
||||
continue
|
||||
}
|
||||
|
||||
sum, err := csaf.NewAdvisorySummary(w.expr, advisory)
|
||||
if err != nil {
|
||||
log.Printf("error: %s: %v\n", file, err)
|
||||
continue
|
||||
}
|
||||
summaries = append(summaries, summary{
|
||||
filename: filename,
|
||||
summary: sum,
|
||||
url: file,
|
||||
})
|
||||
|
||||
year := sum.InitialReleaseDate.Year()
|
||||
|
||||
yearDir := yearDirs[year]
|
||||
if yearDir == "" {
|
||||
yearDir = filepath.Join(dir, label, strconv.Itoa(year))
|
||||
if err := os.MkdirAll(yearDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
//log.Printf("created %s\n", yearDir)
|
||||
yearDirs[year] = yearDir
|
||||
}
|
||||
|
||||
fname := filepath.Join(yearDir, filename)
|
||||
//log.Printf("write: %s\n", fname)
|
||||
data := content.Bytes()
|
||||
if err := writeFileHashes(
|
||||
fname, filename,
|
||||
data, s256.Sum(nil), s512.Sum(nil),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Try to fetch signature file.
|
||||
sigURL := file + ".asc"
|
||||
ascFile := fname + ".asc"
|
||||
if err := w.downloadSignatureOrSign(sigURL, ascFile, data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
w.summaries[label] = summaries
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadSignatureOrSign first tries to download a signature.
|
||||
// If this fails it creates a signature itself with the configured key.
|
||||
func (w *worker) downloadSignatureOrSign(url, fname string, data []byte) error {
|
||||
sig, err := w.downloadSignature(url)
|
||||
|
||||
if err != nil {
|
||||
if err != errNotFound {
|
||||
log.Printf("error: %s: %v\n", url, err)
|
||||
}
|
||||
// Sign it our self.
|
||||
if sig, err = w.sign(data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if sig != "" {
|
||||
err = os.WriteFile(fname, []byte(sig), 0644)
|
||||
}
|
||||
return err
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue