SystemSh0cker/gocsaf
1
0
Fork 0
mirror of https://github.com/gocsaf/csaf.git synced 2025-12-22 05:40:11 +01:00
Code Issues Releases Activity Actions

Merge branch 'main' into csaf_searcher

Browse source
This commit is contained in:
JanHoefelmeyer 2023-10-18 10:24:34 +02:00
commit 226dc961f3
68 changed files with 1726 additions and 246 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/csaf_aggregator/client.go
View file

@ -13,7 +13,7 @@ import (
"io"
"net/http"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
var errNotFound = errors.New("not found")

12
cmd/csaf_aggregator/config.go
View file

@ -19,12 +19,12 @@ import (
"time"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/internal/certs"
"github.com/csaf-poc/csaf_distribution/v2/internal/filter"
"github.com/csaf-poc/csaf_distribution/v2/internal/models"
"github.com/csaf-poc/csaf_distribution/v2/internal/options"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
"github.com/csaf-poc/csaf_distribution/v3/internal/filter"
"github.com/csaf-poc/csaf_distribution/v3/internal/models"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/util"
"golang.org/x/time/rate"
)

4
cmd/csaf_aggregator/full.go
View file

@ -18,8 +18,8 @@ import (
"sync"
"time"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type fullJob struct {

4
cmd/csaf_aggregator/indices.go
View file

@ -20,8 +20,8 @@ import (
"strings"
"time"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
const (

4
cmd/csaf_aggregator/interim.go
View file

@ -25,8 +25,8 @@ import (
"sync"
"time"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type interimJob struct {

2
cmd/csaf_aggregator/lazytransaction.go
View file

@ -13,7 +13,7 @@ import (
"os"
"path/filepath"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type lazyTransaction struct {

4
cmd/csaf_aggregator/lister.go
View file

@ -11,8 +11,8 @@ package main
import (
"fmt"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
// mirrorAllowed checks if mirroring is allowed.

2
cmd/csaf_aggregator/main.go
View file

@ -14,7 +14,7 @@ import (
"os"
"path/filepath"
"github.com/csaf-poc/csaf_distribution/v2/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
"github.com/gofrs/flock"
)

4
cmd/csaf_aggregator/mirror.go
View file

@ -30,8 +30,8 @@ import (
"github.com/ProtonMail/gopenpgp/v2/constants"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
// mirrorAllowed checks if mirroring is allowed.

5
cmd/csaf_aggregator/processor.go
View file

@ -15,8 +15,9 @@ import (
"path/filepath"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type processor struct {

8
cmd/csaf_checker/config.go
View file

@ -13,10 +13,10 @@ import (
"fmt"
"net/http"
"github.com/csaf-poc/csaf_distribution/v2/internal/certs"
"github.com/csaf-poc/csaf_distribution/v2/internal/filter"
"github.com/csaf-poc/csaf_distribution/v2/internal/models"
"github.com/csaf-poc/csaf_distribution/v2/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
"github.com/csaf-poc/csaf_distribution/v3/internal/filter"
"github.com/csaf-poc/csaf_distribution/v3/internal/models"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)
type outputFormat string

3
cmd/csaf_checker/links.go
View file

@ -14,7 +14,8 @@ import (
"net/url"
"github.com/PuerkitoBio/goquery"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type (

8
cmd/csaf_checker/links_test.go
View file

@ -1,3 +1,11 @@
// This file is Free Software under the MIT License
// without warranty, see README.md and LICENSES/MIT.txt for details.
//
// SPDX-License-Identifier: MIT
//
// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
package main
import (

2
cmd/csaf_checker/main.go
View file

@ -12,7 +12,7 @@ package main
import (
"log"
"github.com/csaf-poc/csaf_distribution/v2/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)
// run uses a processor to check all the given domains or direct urls

33
cmd/csaf_checker/processor.go
View file

@ -32,8 +32,9 @@ import (
"github.com/ProtonMail/gopenpgp/v2/crypto"
"golang.org/x/time/rate"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/internal/models"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
// topicMessages stores the collected topicMessages for a specific topic.
@ -666,12 +667,9 @@ func (p *processor) integrity(
var folderYear *int
if m := yearFromURL.FindStringSubmatch(u); m != nil {
year, _ := strconv.Atoi(m[1])
// Check if we are in checking time interval.
if accept := p.cfg.Range; accept != nil && !accept.Contains(
time.Date(
year, 12, 31, // Assume last day of year.
23, 59, 59, 0, // 23:59:59
time.UTC)) {
// Check if the year is in the accepted time interval.
if accept := p.cfg.Range; accept != nil &&
!accept.Intersects(models.Year(year)) {
continue
}
folderYear = &year
@ -1115,6 +1113,8 @@ func (p *processor) checkMissing(string) error {
for _, f := range files {
v := p.alreadyChecked[f]
var where []string
// mistake contains which requirements are broken
var mistake whereType
for mask := rolieMask; mask <= listingMask; mask <<= 1 {
if maxMask&mask == mask {
var in string
@ -1122,11 +1122,26 @@ func (p *processor) checkMissing(string) error {
in = "in"
} else {
in = "not in"
// Which file is missing entries?
mistake |= mask
}
where = append(where, in+" "+mask.String())
}
}
p.badIntegrities.error("%s %s", f, strings.Join(where, ", "))
// List error in all appropriate categories
if mistake&(rolieMask|indexMask|changesMask|listingMask) == 0 {
continue
}
joined := strings.Join(where, ", ")
report := func(mask whereType, msgs *topicMessages) {
if mistake&mask != 0 {
msgs.error("%s %s", f, joined)
}
}
report(rolieMask, &p.badROLIEFeed)
report(indexMask, &p.badIndices)
report(changesMask, &p.badChanges)
report(listingMask, &p.badDirListings)
}
return nil
}

4
cmd/csaf_checker/report.go
View file

@ -18,8 +18,8 @@ import (
"os"
"time"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/internal/models"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/internal/models"
)
// MessageType is the kind of the message.

2
cmd/csaf_checker/reporters.go
View file

@ -13,7 +13,7 @@ import (
"sort"
"strings"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type (

4
cmd/csaf_checker/roliecheck.go
View file

@ -15,8 +15,8 @@ import (
"sort"
"strings"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
// identifier consist of document/tracking/id and document/publisher/namespace,

2
cmd/csaf_checker/rules.go
View file

@ -12,7 +12,7 @@ import (
"fmt"
"sort"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
)
type ruleCondition int

85
cmd/csaf_downloader/config.go
View file

@ -12,16 +12,17 @@ import (
"crypto/tls"
"fmt"
"io"
"log"
"log/slog"
"net/http"
"os"
"path/filepath"
"time"
"github.com/csaf-poc/csaf_distribution/v2/internal/certs"
"github.com/csaf-poc/csaf_distribution/v2/internal/filter"
"github.com/csaf-poc/csaf_distribution/v2/internal/models"
"github.com/csaf-poc/csaf_distribution/v2/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
"github.com/csaf-poc/csaf_distribution/v3/internal/filter"
"github.com/csaf-poc/csaf_distribution/v3/internal/models"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)
const (
@ -30,7 +31,7 @@ const (
defaultForwardQueue = 5
defaultValidationMode = validationStrict
defaultLogFile = "downloader.log"
defaultLogLevel = logLevelInfo
defaultLogLevel = slog.LevelInfo
)
type validationMode string
@ -40,15 +41,6 @@ const (
validationUnsafe = validationMode("unsafe")
)
type logLevel string
const (
logLevelDebug = logLevel("debug")
logLevelInfo = logLevel("info")
logLevelWarn = logLevel("warn")
logLevelError = logLevel("error")
)
type config struct {
Directory string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"`
@ -57,7 +49,6 @@ type config struct {
ClientKey *string `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE" toml:"client_key"`
ClientPassphrase *string `long:"client-passphrase" description:"Optional passphrase for the client cert (limited, experimental, see doc)" value-name:"PASSPHRASE" toml:"client_passphrase"`
Version bool `long:"version" description:"Display version of the binary" toml:"-"`
Verbose bool `long:"verbose" short:"v" description:"Verbose output" toml:"verbose"`
NoStore bool `long:"nostore" short:"n" description:"Do not store files" toml:"no_store"`
Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)" toml:"rate"`
Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"`
@ -78,9 +69,9 @@ type config struct {
ForwardQueue int `long:"forwardqueue" description:"Maximal queue LENGTH before forwarder" value-name:"LENGTH" toml:"forward_queue"`
ForwardInsecure bool `long:"forwardinsecure" description:"Do not check TLS certificates from forward endpoint" toml:"forward_insecure"`
LogFile string `long:"logfile" description:"FILE to log downloading to" value-name:"FILE" toml:"log_file"`
LogFile *string `long:"logfile" description:"FILE to log downloading to" value-name:"FILE" toml:"log_file"`
//lint:ignore SA5008 We are using choice or than once: debug, info, warn, error
LogLevel logLevel `long:"loglevel" description:"LEVEL of logging details" value-name:"LEVEL" choice:"debug" choice:"info" choice:"warn" choice:"error" toml:"log_level"`
LogLevel *options.LogLevel `long:"loglevel" description:"LEVEL of logging details" value-name:"LEVEL" choice:"debug" choice:"info" choice:"warn" choice:"error" toml:"log_level"`
Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"`
@ -97,6 +88,10 @@ var configPaths = []string{
// parseArgsConfig parses the command line and if need a config file.
func parseArgsConfig() ([]string, *config, error) {
var (
logFile = defaultLogFile
logLevel = &options.LogLevel{Level: defaultLogLevel}
)
p := options.Parser[config]{
DefaultConfigLocations: configPaths,
ConfigLocation: func(cfg *config) string { return cfg.Config },
@ -107,8 +102,8 @@ func parseArgsConfig() ([]string, *config, error) {
cfg.RemoteValidatorPresets = []string{defaultPreset}
cfg.ValidationMode = defaultValidationMode
cfg.ForwardQueue = defaultForwardQueue
cfg.LogFile = defaultLogFile
cfg.LogLevel = defaultLogLevel
cfg.LogFile = &logFile
cfg.LogLevel = logLevel
},
// Re-establish default values if not set.
EnsureDefaults: func(cfg *config) {
@ -123,30 +118,35 @@ func parseArgsConfig() ([]string, *config, error) {
default:
cfg.ValidationMode = validationStrict
}
if cfg.LogFile == nil {
cfg.LogFile = &logFile
}
if cfg.LogLevel == nil {
cfg.LogLevel = logLevel
}
},
}
return p.Parse()
}
// UnmarshalText implements [encoding/text.TextUnmarshaler].
// UnmarshalText implements [encoding.TextUnmarshaler].
func (vm *validationMode) UnmarshalText(text []byte) error {
switch m := validationMode(text); m {
case validationStrict, validationUnsafe:
*vm = m
default:
return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe"`, m)
return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe)"`, m)
}
return nil
}
// UnmarshalText implements [encoding/text.TextUnmarshaler].
func (ll *logLevel) UnmarshalText(text []byte) error {
switch l := logLevel(text); l {
case logLevelDebug, logLevelInfo, logLevelWarn, logLevelError:
*ll = l
default:
return fmt.Errorf(`invalid value %q (expected "debug", "info", "warn", "error")`, l)
// UnmarshalFlag implements [flags.UnmarshalFlag].
func (vm *validationMode) UnmarshalFlag(value string) error {
var v validationMode
if err := v.UnmarshalText([]byte(value)); err != nil {
return err
}
*vm = v
return nil
}
@ -155,20 +155,9 @@ func (cfg *config) ignoreURL(u string) bool {
return cfg.ignorePattern.Matches(u)
}
// slogLevel converts logLevel to [slog.Level].
func (ll logLevel) slogLevel() slog.Level {
switch ll {
case logLevelDebug:
return slog.LevelDebug
case logLevelInfo:
return slog.LevelInfo
case logLevelWarn:
return slog.LevelWarn
case logLevelError:
return slog.LevelError
default:
return slog.LevelInfo
}
// verbose is considered a log level equal or less debug.
func (cfg *config) verbose() bool {
return cfg.LogLevel.Level <= slog.LevelDebug
}
// prepareDirectory ensures that the working directory
@ -209,26 +198,28 @@ func dropSubSeconds(_ []string, a slog.Attr) slog.Attr {
// prepareLogging sets up the structured logging.
func (cfg *config) prepareLogging() error {
var w io.Writer
if cfg.LogFile == "" {
if cfg.LogFile == nil || *cfg.LogFile == "" {
log.Println("using STDERR for logging")
w = os.Stderr
} else {
var fname string
// We put the log inside the download folder
// if it is not absolute.
if filepath.IsAbs(cfg.LogFile) {
fname = cfg.LogFile
if filepath.IsAbs(*cfg.LogFile) {
fname = *cfg.LogFile
} else {
fname = filepath.Join(cfg.Directory, cfg.LogFile)
fname = filepath.Join(cfg.Directory, *cfg.LogFile)
}
f, err := os.OpenFile(fname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
if err != nil {
return err
}
log.Printf("using %q for logging\n", *cfg.LogFile)
w = f
}
ho := slog.HandlerOptions{
//AddSource: true,
Level: cfg.LogLevel.slogLevel(),
Level: cfg.LogLevel.Level,
ReplaceAttr: dropSubSeconds,
}
handler := slog.NewJSONHandler(w, &ho)

71
cmd/csaf_downloader/downloader.go
View file

@ -33,8 +33,8 @@ import (
"github.com/ProtonMail/gopenpgp/v2/crypto"
"golang.org/x/time/rate"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type downloader struct {
@ -92,10 +92,26 @@ func (d *downloader) addStats(o *stats) {
d.stats.add(o)
}
// logRedirect logs redirects of the http client.
func logRedirect(req *http.Request, via []*http.Request) error {
vs := make([]string, len(via))
for i, v := range via {
vs[i] = v.URL.String()
}
slog.Debug("Redirecting",
"to", req.URL.String(),
"via", strings.Join(vs, " -> "))
return nil
}
func (d *downloader) httpClient() util.Client {
hClient := http.Client{}
if d.cfg.verbose() {
hClient.CheckRedirect = logRedirect
}
var tlsConfig tls.Config
if d.cfg.Insecure {
tlsConfig.InsecureSkipVerify = true
@ -120,8 +136,11 @@ func (d *downloader) httpClient() util.Client {
}
// Add optional URL logging.
if d.cfg.Verbose {
client = &util.LoggingClient{Client: client}
if d.cfg.verbose() {
client = &util.LoggingClient{
Client: client,
Log: httpLog("downloader"),
}
}
// Add optional rate limiting.
@ -135,6 +154,16 @@ func (d *downloader) httpClient() util.Client {
return client
}
// httpLog does structured logging in a [util.LoggingClient].
func httpLog(who string) func(string, string) {
return func(method, url string) {
slog.Debug("http",
"who", who,
"method", method,
"url", url)
}
}
func (d *downloader) download(ctx context.Context, domain string) error {
client := d.httpClient()
@ -142,9 +171,9 @@ func (d *downloader) download(ctx context.Context, domain string) error {
lpmd := loader.Load(domain)
if d.cfg.Verbose {
if d.cfg.verbose() {
for i := range lpmd.Messages {
slog.Info("Loading provider-metadata.json",
slog.Debug("Loading provider-metadata.json",
"domain", domain,
"message", lpmd.Messages[i].Message)
}
@ -331,7 +360,7 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error)
return
}
if len(errors) > 0 {
if d.cfg.Verbose {
if d.cfg.verbose() {
slog.Error("CSAF file has validation errors",
"url", url,
"error", strings.Join(errors, ", "))
@ -388,9 +417,7 @@ nextAdvisory:
}
if d.cfg.ignoreURL(file.URL()) {
if d.cfg.Verbose {
slog.Warn("Ignoring URL", "url", file.URL())
}
slog.Debug("Ignoring URL", "url", file.URL())
continue
}
@ -438,22 +465,18 @@ nextAdvisory:
// Only hash when we have a remote counter part we can compare it with.
if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
if d.cfg.Verbose {
slog.Warn("Cannot fetch SHA256",
"url", file.SHA256URL(),
"error", err)
}
slog.Warn("Cannot fetch SHA256",
"url", file.SHA256URL(),
"error", err)
} else {
s256 = sha256.New()
writers = append(writers, s256)
}
if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
if d.cfg.Verbose {
slog.Warn("Cannot fetch SHA512",
"url", file.SHA512URL(),
"error", err)
}
slog.Warn("Cannot fetch SHA512",
"url", file.SHA512URL(),
"error", err)
} else {
s512 = sha512.New()
writers = append(writers, s512)
@ -506,11 +529,9 @@ nextAdvisory:
var sign *crypto.PGPSignature
sign, signData, err = loadSignature(client, file.SignURL())
if err != nil {
if d.cfg.Verbose {
slog.Warn("Downloading signature failed",
"url", file.SignURL(),
"error", err)
}
slog.Warn("Downloading signature failed",
"url", file.SignURL(),
"error", err)
}
if sign != nil {
if err := d.checkSignature(data.Bytes(), sign); err != nil {

53
cmd/csaf_downloader/forwarder.go
View file

@ -19,8 +19,8 @@ import (
"path/filepath"
"strings"
"github.com/csaf-poc/csaf_distribution/v2/internal/misc"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/internal/misc"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
// failedForwardDir is the name of the special sub folder
@ -116,8 +116,11 @@ func (f *forwarder) httpClient() util.Client {
}
// Add optional URL logging.
if f.cfg.Verbose {
client = &util.LoggingClient{Client: client}
if f.cfg.verbose() {
client = &util.LoggingClient{
Client: client,
Log: httpLog("forwarder"),
}
}
f.client = client
@ -184,16 +187,10 @@ func (f *forwarder) buildRequest(
// storeFailedAdvisory stores an advisory in a special folder
// in case the forwarding failed.
func (f *forwarder) storeFailedAdvisory(filename, doc, sha256, sha512 string) error {
dir := filepath.Join(f.cfg.Directory, failedForwardDir)
// Create special folder if it does not exist.
if _, err := os.Stat(dir); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
} else {
return err
}
dir := filepath.Join(f.cfg.Directory, failedForwardDir)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
// Store parts which are not empty.
for _, x := range []struct {
@ -223,6 +220,19 @@ func (f *forwarder) storeFailed(filename, doc, sha256, sha512 string) {
}
}
// limitedString reads max bytes from reader and returns it as a string.
// Longer strings are indicated by "..." as a suffix.
func limitedString(r io.Reader, max int) (string, error) {
var msg strings.Builder
if _, err := io.Copy(&msg, io.LimitReader(r, int64(max))); err != nil {
return "", err
}
if msg.Len() >= max {
msg.WriteString("...")
}
return msg.String(), nil
}
// forward sends a given document with filename, status and
// checksums to the forwarder. This is async to the degree
// till the configured queue size is filled.
@ -249,16 +259,15 @@ func (f *forwarder) forward(
}
if res.StatusCode != http.StatusCreated {
defer res.Body.Close()
var msg strings.Builder
io.Copy(&msg, io.LimitReader(res.Body, 512))
var dots string
if msg.Len() >= 512 {
dots = "..."
if msg, err := limitedString(res.Body, 512); err != nil {
slog.Error("reading forward result failed",
"error", err)
} else {
slog.Error("forwarding failed",
"filename", filename,
"body", msg,
"status_code", res.StatusCode)
}
slog.Error("forwarding failed",
"filename", filename,
"body", msg.String()+dots,
"status_code", res.StatusCode)
f.storeFailed(filename, doc, sha256, sha512)
} else {
f.succeeded++

429
cmd/csaf_downloader/forwarder_test.go Normal file
View file

@ -0,0 +1,429 @@
// This file is Free Software under the MIT License
// without warranty, see README.md and LICENSES/MIT.txt for details.
//
// SPDX-License-Identifier: MIT
//
// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
package main
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"io"
"log/slog"
"mime"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
func TestValidationStatusUpdate(t *testing.T) {
sv := validValidationStatus
sv.update(invalidValidationStatus)
sv.update(validValidationStatus)
if sv != invalidValidationStatus {
t.Fatalf("got %q expected %q", sv, invalidValidationStatus)
}
sv = notValidatedValidationStatus
sv.update(validValidationStatus)
sv.update(notValidatedValidationStatus)
if sv != notValidatedValidationStatus {
t.Fatalf("got %q expected %q", sv, notValidatedValidationStatus)
}
}
func TestForwarderLogStats(t *testing.T) {
orig := slog.Default()
defer slog.SetDefault(orig)
var buf bytes.Buffer
h := slog.NewJSONHandler(&buf, &slog.HandlerOptions{
Level: slog.LevelInfo,
})
lg := slog.New(h)
slog.SetDefault(lg)
cfg := &config{}
fw := newForwarder(cfg)
fw.failed = 11
fw.succeeded = 13
done := make(chan struct{})
go func() {
defer close(done)
fw.run()
}()
fw.log()
fw.close()
<-done
type fwStats struct {
Msg string `json:"msg"`
Succeeded int `json:"succeeded"`
Failed int `json:"failed"`
}
sc := bufio.NewScanner(bytes.NewReader(buf.Bytes()))
found := false
for sc.Scan() {
var fws fwStats
if err := json.Unmarshal(sc.Bytes(), &fws); err != nil {
t.Fatalf("JSON parsing log failed: %v", err)
}
if fws.Msg == "Forward statistics" &&
fws.Failed == 11 &&
fws.Succeeded == 13 {
found = true
break
}
}
if err := sc.Err(); err != nil {
t.Fatalf("scanning log failed: %v", err)
}
if !found {
t.Fatal("Cannot find forward statistics in log")
}
}
func TestForwarderHTTPClient(t *testing.T) {
cfg := &config{
ForwardInsecure: true,
ForwardHeader: http.Header{
"User-Agent": []string{"curl/7.55.1"},
},
LogLevel: &options.LogLevel{Level: slog.LevelDebug},
}
fw := newForwarder(cfg)
if c1, c2 := fw.httpClient(), fw.httpClient(); c1 != c2 {
t.Fatal("expected to return same client twice")
}
}
func TestForwarderReplaceExtension(t *testing.T) {
for _, x := range [][2]string{
{"foo", "foo.ext"},
{"foo.bar", "foo.ext"},
{".bar", ".ext"},
{"", ".ext"},
} {
if got := replaceExt(x[0], ".ext"); got != x[1] {
t.Fatalf("got %q expected %q", got, x[1])
}
}
}
func TestForwarderBuildRequest(t *testing.T) {
// Good case ...
cfg := &config{
ForwardURL: "https://example.com",
}
fw := newForwarder(cfg)
req, err := fw.buildRequest(
"test.json", "{}",
invalidValidationStatus,
"256",
"512")
if err != nil {
t.Fatalf("buildRequest failed: %v", err)
}
mediaType, params, err := mime.ParseMediaType(req.Header.Get("Content-Type"))
if err != nil {
t.Fatalf("no Content-Type found")
}
if !strings.HasPrefix(mediaType, "multipart/") {
t.Fatalf("media type is not multipart")
}
mr := multipart.NewReader(req.Body, params["boundary"])
var foundAdvisory, foundValidationStatus, found256, found512 bool
for {
p, err := mr.NextPart()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("parsing multipart failed: %v", err)
}
data, err := io.ReadAll(p)
if err != nil {
t.Fatal(err)
}
cd := p.Header["Content-Disposition"]
if len(cd) == 0 {
continue
}
switch contains := func(name string) bool {
return strings.Contains(cd[0], `name="`+name+`"`)
}; {
case contains("advisory"):
if a := string(data); a != "{}" {
t.Fatalf("advisory: got %q expected %q", a, "{}")
}
foundAdvisory = true
case contains("validation_status"):
if vs := validationStatus(data); vs != invalidValidationStatus {
t.Fatalf("validation_status: got %q expected %q",
vs, invalidValidationStatus)
}
foundValidationStatus = true
case contains("hash-256"):
if h := string(data); h != "256" {
t.Fatalf("hash-256: got %q expected %q", h, "256")
}
found256 = true
case contains("hash-512"):
if h := string(data); h != "512" {
t.Fatalf("hash-512: got %q expected %q", h, "512")
}
found512 = true
}
}
switch {
case !foundAdvisory:
t.Fatal("advisory not found")
case !foundValidationStatus:
t.Fatal("validation_status not found")
case !found256:
t.Fatal("hash-256 not found")
case !found512:
t.Fatal("hash-512 not found")
}
// Bad case ...
cfg.ForwardURL = "%"
if _, err := fw.buildRequest(
"test.json", "{}",
invalidValidationStatus,
"256",
"512",
); err == nil {
t.Fatal("bad forward URL should result in an error")
}
}
type badReader struct{ error }
func (br *badReader) Read([]byte) (int, error) { return 0, br.error }
func TestLimitedString(t *testing.T) {
for _, x := range [][2]string{
{"xx", "xx"},
{"xxx", "xxx..."},
{"xxxx", "xxx..."},
} {
got, err := limitedString(strings.NewReader(x[0]), 3)
if err != nil {
t.Fatal(err)
}
if got != x[1] {
t.Fatalf("got %q expected %q", got, x[1])
}
}
if _, err := limitedString(&badReader{error: os.ErrInvalid}, 3); err == nil {
t.Fatal("expected to fail with an error")
}
}
func TestStoreFailedAdvisory(t *testing.T) {
dir, err := os.MkdirTemp("", "storeFailedAdvisory")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
cfg := &config{Directory: dir}
fw := newForwarder(cfg)
badDir := filepath.Join(dir, failedForwardDir)
if err := os.WriteFile(badDir, []byte("test"), 0664); err != nil {
t.Fatal(err)
}
if err := fw.storeFailedAdvisory("advisory.json", "{}", "256", "512"); err == nil {
t.Fatal("if the destination exists as a file an error should occur")
}
if err := os.Remove(badDir); err != nil {
t.Fatal(err)
}
if err := fw.storeFailedAdvisory("advisory.json", "{}", "256", "512"); err != nil {
t.Fatal(err)
}
sha256Path := filepath.Join(dir, failedForwardDir, "advisory.json.sha256")
// Write protect advisory.
if err := os.Chmod(sha256Path, 0); err != nil {
t.Fatal(err)
}
if err := fw.storeFailedAdvisory("advisory.json", "{}", "256", "512"); err == nil {
t.Fatal("expected to fail with an error")
}
if err := os.Chmod(sha256Path, 0644); err != nil {
t.Fatal(err)
}
}
func TestStoredFailed(t *testing.T) {
dir, err := os.MkdirTemp("", "storeFailed")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
orig := slog.Default()
defer slog.SetDefault(orig)
var buf bytes.Buffer
h := slog.NewJSONHandler(&buf, &slog.HandlerOptions{
Level: slog.LevelError,
})
lg := slog.New(h)
slog.SetDefault(lg)
cfg := &config{Directory: dir}
fw := newForwarder(cfg)
// An empty filename should lead to an error.
fw.storeFailed("", "{}", "256", "512")
if fw.failed != 1 {
t.Fatalf("got %d expected 1", fw.failed)
}
type entry struct {
Msg string `json:"msg"`
Level string `json:"level"`
}
sc := bufio.NewScanner(bytes.NewReader(buf.Bytes()))
found := false
for sc.Scan() {
var e entry
if err := json.Unmarshal(sc.Bytes(), &e); err != nil {
t.Fatalf("JSON parsing log failed: %v", err)
}
if e.Msg == "Storing advisory failed forwarding failed" && e.Level == "ERROR" {
found = true
break
}
}
if err := sc.Err(); err != nil {
t.Fatalf("scanning log failed: %v", err)
}
if !found {
t.Fatal("Cannot error logging statistics in log")
}
}
type fakeClient struct {
util.Client
state int
}
func (fc *fakeClient) Do(*http.Request) (*http.Response, error) {
// The different states simulates different responses from the remote API.
switch fc.state {
case 0:
fc.state = 1
return &http.Response{
Status: http.StatusText(http.StatusCreated),
StatusCode: http.StatusCreated,
}, nil
case 1:
fc.state = 2
return nil, errors.New("does not work")
case 2:
fc.state = 3
return &http.Response{
Status: http.StatusText(http.StatusBadRequest),
StatusCode: http.StatusBadRequest,
Body: io.NopCloser(&badReader{error: os.ErrInvalid}),
}, nil
default:
return &http.Response{
Status: http.StatusText(http.StatusBadRequest),
StatusCode: http.StatusBadRequest,
Body: io.NopCloser(strings.NewReader("This was bad!")),
}, nil
}
}
func TestForwarderForward(t *testing.T) {
dir, err := os.MkdirTemp("", "forward")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
orig := slog.Default()
defer slog.SetDefault(orig)
// We dont care in details here as we captured them
// in the other test cases.
h := slog.NewJSONHandler(io.Discard, nil)
lg := slog.New(h)
slog.SetDefault(lg)
cfg := &config{
ForwardURL: "http://example.com",
Directory: dir,
}
fw := newForwarder(cfg)
// Use the fact that http client is cached.
fw.client = &fakeClient{}
done := make(chan struct{})
go func() {
defer close(done)
fw.run()
}()
// Iterate through states of http client.
for i := 0; i <= 3; i++ {
fw.forward(
"test.json", "{}",
invalidValidationStatus,
"256",
"512")
}
// Make buildRequest fail.
wait := make(chan struct{})
fw.cmds <- func(f *forwarder) {
f.cfg.ForwardURL = "%"
close(wait)
}
<-wait
fw.forward(
"test.json", "{}",
invalidValidationStatus,
"256",
"512")
fw.close()
<-done
}

2
cmd/csaf_downloader/main.go
View file

@ -15,7 +15,7 @@ import (
"os"
"os/signal"
"github.com/csaf-poc/csaf_distribution/v2/internal/options"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)
func run(cfg *config, domains []string) error {

112
cmd/csaf_downloader/stats_test.go Normal file
View file

@ -0,0 +1,112 @@
// This file is Free Software under the MIT License
// without warranty, see README.md and LICENSES/MIT.txt for details.
//
// SPDX-License-Identifier: MIT
//
// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
package main
import (
"bytes"
"encoding/json"
"log/slog"
"testing"
)
func TestStatsAdd(t *testing.T) {
a := stats{
downloadFailed: 2,
filenameFailed: 3,
schemaFailed: 5,
remoteFailed: 7,
sha256Failed: 11,
sha512Failed: 13,
signatureFailed: 17,
succeeded: 19,
}
b := a
a.add(&b)
b.downloadFailed *= 2
b.filenameFailed *= 2
b.schemaFailed *= 2
b.remoteFailed *= 2
b.sha256Failed *= 2
b.sha512Failed *= 2
b.signatureFailed *= 2
b.succeeded *= 2
if a != b {
t.Fatalf("%v != %v", a, b)
}
}
func TestStatsTotalFailed(t *testing.T) {
a := stats{
downloadFailed: 2,
filenameFailed: 3,
schemaFailed: 5,
remoteFailed: 7,
sha256Failed: 11,
sha512Failed: 13,
signatureFailed: 17,
}
sum := a.downloadFailed +
a.filenameFailed +
a.schemaFailed +
a.remoteFailed +
a.sha256Failed +
a.sha512Failed +
a.signatureFailed
if got := a.totalFailed(); got != sum {
t.Fatalf("got %d expected %d", got, sum)
}
}
func TestStatsLog(t *testing.T) {
var out bytes.Buffer
h := slog.NewJSONHandler(&out, &slog.HandlerOptions{Level: slog.LevelInfo})
orig := slog.Default()
defer slog.SetDefault(orig)
slog.SetDefault(slog.New(h))
a := stats{
downloadFailed: 2,
filenameFailed: 3,
schemaFailed: 5,
remoteFailed: 7,
sha256Failed: 11,
sha512Failed: 13,
signatureFailed: 17,
succeeded: 19,
}
a.log()
type result struct {
Succeeded int `json:"succeeded"`
TotalFailed int `json:"total_failed"`
FilenameFailed int `json:"filename_failed"`
DownloadFailed int `json:"download_failed"`
SchemaFailed int `json:"schema_failed"`
RemoteFailed int `json:"remote_failed"`
SHA256Failed int `json:"sha256_failed"`
SHA512Failed int `json:"sha512_failed"`
SignatureFailed int `json:"signature_failed"`
}
var got result
if err := json.Unmarshal(out.Bytes(), &got); err != nil {
t.Fatal(err)
}
want := result{
Succeeded: a.succeeded,
TotalFailed: a.totalFailed(),
FilenameFailed: a.filenameFailed,
DownloadFailed: a.downloadFailed,
SchemaFailed: a.schemaFailed,
RemoteFailed: a.remoteFailed,
SHA256Failed: a.sha256Failed,
SHA512Failed: a.sha512Failed,
SignatureFailed: a.signatureFailed,
}
if got != want {
t.Fatalf("%v != %v", got, want)
}
}

5
cmd/csaf_provider/actions.go
View file

@ -25,8 +25,9 @@ import (
"github.com/ProtonMail/gopenpgp/v2/armor"
"github.com/ProtonMail/gopenpgp/v2/constants"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
const dateFormat = time.RFC3339

3
cmd/csaf_provider/config.go
View file

@ -16,8 +16,9 @@ import (
"github.com/BurntSushi/toml"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"golang.org/x/crypto/bcrypt"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
)
const (

5
cmd/csaf_provider/create.go
View file

@ -21,8 +21,9 @@ import (
"unicode"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
// ensureFolders initializes the paths and call functions to create

2
cmd/csaf_provider/files.go
View file

@ -13,7 +13,7 @@ import (
"crypto/sha512"
"os"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
func writeHashedFile(fname, name string, data []byte, armored string) error {

2
cmd/csaf_provider/indices.go
View file

@ -18,7 +18,7 @@ import (
"sort"
"time"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
func updateIndex(dir, fname string) error {

3
cmd/csaf_provider/main.go
View file

@ -16,8 +16,9 @@ import (
"net/http/cgi"
"os"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/jessevdk/go-flags"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type options struct {

4
cmd/csaf_provider/rolie.go
View file

@ -15,8 +15,8 @@ import (
"strings"
"time"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
// mergeCategories merges the given categories into the old ones.

4
cmd/csaf_provider/transaction.go
View file

@ -12,8 +12,8 @@ import (
"os"
"path/filepath"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
func doTransaction(

5
cmd/csaf_uploader/config.go
View file

@ -15,10 +15,11 @@ import (
"os"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/internal/certs"
"github.com/csaf-poc/csaf_distribution/v2/internal/options"
"golang.org/x/crypto/bcrypt"
"golang.org/x/term"
"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)
const (

2
cmd/csaf_uploader/main.go
View file

@ -9,7 +9,7 @@
// Implements a command line tool that uploads csaf documents to csaf_provider.
package main
import "github.com/csaf-poc/csaf_distribution/v2/internal/options"
import "github.com/csaf-poc/csaf_distribution/v3/internal/options"
func main() {
args, cfg, err := parseArgsConfig()

6
cmd/csaf_uploader/processor.go
View file

@ -26,9 +26,9 @@ import (
"github.com/ProtonMail/gopenpgp/v2/constants"
"github.com/ProtonMail/gopenpgp/v2/crypto"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/internal/misc"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/internal/misc"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type processor struct {

5
cmd/csaf_validator/main.go
View file

@ -16,9 +16,10 @@ import (
"os"
"path/filepath"
"github.com/csaf-poc/csaf_distribution/v2/csaf"
"github.com/csaf-poc/csaf_distribution/v2/util"
"github.com/jessevdk/go-flags"
"github.com/csaf-poc/csaf_distribution/v3/csaf"
"github.com/csaf-poc/csaf_distribution/v3/util"
)
type options struct {