Merge branch 'sha-handling' into unittest

cmd/csaf_checker/processor.go

@@ -84,10 +84,8 @@ type reporter interface {
 	report(*processor, *Domain)
 }
 
-var (
 // errContinue indicates that the current check should continue.
-	errContinue = errors.New("continue")
+var errContinue = errors.New("continue")
-)
 
 type whereType byte
 
@@ -167,7 +165,6 @@ func (m *topicMessages) hasErrors() bool {
 
 // newProcessor returns an initialized processor.
 func newProcessor(cfg *config) (*processor, error) {
-
 	var validator csaf.RemoteValidator
 
 	if cfg.RemoteValidator != "" {
@@ -240,7 +237,6 @@ func (p *processor) reset() {
 // Then it calls the report method on each report from the given "reporters" parameter for each domain.
 // It returns a pointer to the report and nil, otherwise an error.
 func (p *processor) run(domains []string) (*Report, error) {
-
 	report := Report{
 		Date:      ReportTime{Time: time.Now().UTC()},
 		Version:   util.SemVersion,
@@ -297,7 +293,6 @@ func (p *processor) run(domains []string) (*Report, error) {
 
 // fillMeta fills the report with extra informations from provider metadata.
 func (p *processor) fillMeta(domain *Domain) error {
-
 	if p.pmd == nil {
 		return nil
 	}
@@ -323,7 +318,6 @@ func (p *processor) fillMeta(domain *Domain) error {
 // domainChecks compiles a list of checks which should be performed
 // for a given domain.
 func (p *processor) domainChecks(domain string) []func(*processor, string) error {
-
 	// If we have a direct domain url we dont need to
 	// perform certain checks.
 	direct := strings.HasPrefix(domain, "https://")
@@ -393,7 +387,6 @@ func (p *processor) markChecked(s string, mask whereType) bool {
 }
 
 func (p *processor) checkRedirect(r *http.Request, via []*http.Request) error {
-
 	url := r.URL.String()
 	p.checkTLS(url)
 	if p.redirects == nil {
@@ -495,7 +488,6 @@ func (p *processor) usedAuthorizedClient() bool {
 
 // rolieFeedEntries loads the references to the advisory files for a given feed.
 func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
-
 	client := p.httpClient()
 	res, err := client.Get(feed)
 	p.badDirListings.use()
@@ -546,7 +538,6 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
 	var files []csaf.AdvisoryFile
 
 	rfeed.Entries(func(entry *csaf.Entry) {
-
 		// Filter if we have date checking.
 		if accept := p.cfg.Range; accept != nil {
 			if t := time.Time(entry.Updated); !t.IsZero() && !accept.Contains(t) {
@@ -759,14 +750,20 @@ func (p *processor) integrity(
 		// Check hashes
 		p.badIntegrities.use()
 
-		for _, x := range []struct {
+		type hash struct {
 			ext  string
 			url  func() string
 			hash []byte
-		}{
+		}
-			{"SHA256", f.SHA256URL, s256.Sum(nil)},
+		hashes := []hash{}
-			{"SHA512", f.SHA512URL, s512.Sum(nil)},
+		if f.SHA256URL() != "" {
-		} {
+			hashes = append(hashes, hash{"SHA256", f.SHA256URL, s256.Sum(nil)})
+		}
+		if f.SHA512URL() != "" {
+			hashes = append(hashes, hash{"SHA512", f.SHA512URL, s512.Sum(nil)})
+		}
+
+		for _, x := range hashes {
 			hu, err := url.Parse(x.url())
 			if err != nil {
 				lg(ErrorType, "Bad URL %s: %v", x.url(), err)
@@ -918,7 +915,6 @@ func (p *processor) checkIndex(base string, mask whereType) error {
 // of the fields' values and if they are sorted properly. Then it passes the files to the
 // "integrity" functions. It returns error if some test fails, otherwise nil.
 func (p *processor) checkChanges(base string, mask whereType) error {
-
 	bu, err := url.Parse(base)
 	if err != nil {
 		return err
@@ -978,8 +974,7 @@ func (p *processor) checkChanges(base string, mask whereType) error {
 			}
 			path := r[pathColumn]
 
-			times, files =
+			times, files = append(times, t),
-				append(times, t),
 				append(files, csaf.DirectoryAdvisoryFile{Path: path})
 		}
 		return times, files, nil
@@ -1152,7 +1147,6 @@ func (p *processor) checkMissing(string) error {
 // checkInvalid goes over all found adivisories URLs and checks
 // if file name conforms to standard.
 func (p *processor) checkInvalid(string) error {
-
 	p.badDirListings.use()
 	var invalids []string
 
@@ -1174,7 +1168,6 @@ func (p *processor) checkInvalid(string) error {
 // checkListing goes over all found adivisories URLs and checks
 // if their parent directory is listable.
 func (p *processor) checkListing(string) error {
-
 	p.badDirListings.use()
 
 	pgs := pages{}
@@ -1209,7 +1202,6 @@ func (p *processor) checkListing(string) error {
 // checkWhitePermissions checks if the TLP:WHITE advisories are
 // available with unprotected access.
 func (p *processor) checkWhitePermissions(string) error {
-
 	var ids []string
 	for id, open := range p.labelChecker.whiteAdvisories {
 		if !open {
@@ -1235,7 +1227,6 @@ func (p *processor) checkWhitePermissions(string) error {
 // According to the result, the respective error messages added to
 // badProviderMetadata.
 func (p *processor) checkProviderMetadata(domain string) bool {
-
 	p.badProviderMetadata.use()
 
 	client := p.httpClient()
@@ -1282,7 +1273,6 @@ func (p *processor) checkSecurity(domain string, legacy bool) (int, string) {
 
 // checkSecurityFolder checks the security.txt in a given folder.
 func (p *processor) checkSecurityFolder(folder string) string {
-
 	client := p.httpClient()
 	path := folder + "security.txt"
 	res, err := client.Get(path)
@@ -1349,7 +1339,6 @@ func (p *processor) checkSecurityFolder(folder string) string {
 // and serves the "provider-metadata.json".
 // It returns an empty string if all checks are passed, otherwise the errormessage.
 func (p *processor) checkDNS(domain string) string {
-
 	client := p.httpClient()
 	path := "https://csaf.data.security." + domain
 	res, err := client.Get(path)
@@ -1359,7 +1348,6 @@ func (p *processor) checkDNS(domain string) string {
 	if res.StatusCode != http.StatusOK {
 		return fmt.Sprintf("Fetching %s failed. Status code %d (%s)",
 			path, res.StatusCode, res.Status)
-
 	}
 	hash := sha256.New()
 	defer res.Body.Close()
@@ -1378,7 +1366,6 @@ func (p *processor) checkDNS(domain string) string {
 // available under the /.well-known/csaf/ directory. Returns the errormessage if
 // an error was encountered, or an empty string otherwise
 func (p *processor) checkWellknown(domain string) string {
-
 	client := p.httpClient()
 	path := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
 
@@ -1408,7 +1395,6 @@ func (p *processor) checkWellknown(domain string) string {
 // The function returns nil, unless errors outside the checks were found.
 // In that case, errors are returned.
 func (p *processor) checkWellknownSecurityDNS(domain string) error {
-
 	warningsW := p.checkWellknown(domain)
 	// Security check for well known (default) and legacy location
 	warningsS, sDMessage := p.checkSecurity(domain, false)
@@ -1461,7 +1447,6 @@ func (p *processor) checkWellknownSecurityDNS(domain string) error {
 // As a result of these a respective error messages are passed to badPGP method
 // in case of errors. It returns nil if all checks are passed.
 func (p *processor) checkPGPKeys(_ string) error {
-
 	p.badPGPs.use()
 
 	src, err := p.expr.Eval("$.public_openpgp_keys", p.pmd)
@@ -1520,7 +1505,6 @@ func (p *processor) checkPGPKeys(_ string) error {
 			defer res.Body.Close()
 			return crypto.NewKeyFromArmoredReader(res.Body)
 		}()
-
 		if err != nil {
 			p.badPGPs.error("Reading public OpenPGP key %s failed: %v", u, err)
 			continue

cmd/csaf_downloader/downloader.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -37,6 +38,13 @@ import (
 	"github.com/csaf-poc/csaf_distribution/v3/util"
 )
 
+type hashFetchInfo struct {
+	url       string
+	preferred bool
+	warn      bool
+	hashType  hashAlgorithm
+}
+
 type downloader struct {
 	cfg       *config
 	client    *util.Client // Used for testing
@@ -502,35 +510,39 @@ nextAdvisory:
 			signData                   []byte
 		)
 
-		if (d.cfg.PreferredHash != algSha512 || file.SHA512URL() == "") && file.SHA256URL() != "" {
+		hashToFetch := []hashFetchInfo{}
-			// Only hash when we have a remote counterpart we can compare it with.
+		if file.SHA512URL() != "" {
-			if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
+			hashToFetch = append(hashToFetch, hashFetchInfo{
-				if !file.IsDirectory() {
+				url:       file.SHA512URL(),
-					slog.Warn("Cannot fetch SHA256",
+				warn:      true,
-						"url", file.SHA256URL(),
+				hashType:  algSha512,
-						"error", err)
+				preferred: strings.EqualFold(string(d.cfg.PreferredHash), string(algSha512))})
 		} else {
-					slog.Info("SHA256 not present", "file", file.URL())
+			slog.Info("SHA512 not present")
 		}
+		if file.SHA256URL() != "" {
+			hashToFetch = append(hashToFetch, hashFetchInfo{
+				url:       file.SHA256URL(),
+				warn:      true,
+				hashType:  algSha256,
+				preferred: strings.EqualFold(string(d.cfg.PreferredHash), string(algSha256))})
 		} else {
-				s256 = sha256.New()
+			slog.Info("SHA256 not present")
-				writers = append(writers, s256)
+		}
+		if file.IsDirectory() {
+			for i := range hashToFetch {
+				hashToFetch[i].warn = false
 			}
 		}
 
-		if (d.cfg.PreferredHash != algSha256 || file.SHA256URL() == "") && file.SHA512URL() != "" {
+		remoteSHA256, s256Data, remoteSHA512, s512Data = loadHashes(client, hashToFetch)
-			if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
+		if remoteSHA512 != nil {
-				if !file.IsDirectory() {
-					slog.Warn("Cannot fetch SHA512",
-						"url", file.SHA512URL(),
-						"error", err)
-				} else {
-					slog.Info("SHA512 not present", "file", file.URL())
-				}
-			} else {
 			s512 = sha512.New()
 			writers = append(writers, s512)
 		}
+		if remoteSHA256 != nil {
+			s256 = sha256.New()
+			writers = append(writers, s256)
 		}
 
 		// Remember the data as we need to store it to file later.
@@ -761,6 +773,51 @@ func loadSignature(client util.Client, p string) (*crypto.PGPSignature, []byte,
 	return sign, data, nil
 }
 
+func loadHashes(client util.Client, hashes []hashFetchInfo) ([]byte, []byte, []byte, []byte) {
+	var remoteSha256, remoteSha512, sha256Data, sha512Data []byte
+
+	// Load preferred hashes first
+	slices.SortStableFunc(hashes, func(a, b hashFetchInfo) int {
+		if a.preferred == b.preferred {
+			return 0
+		}
+		if a.preferred && !b.preferred {
+			return -1
+		} else {
+			return 1
+		}
+	})
+	for _, h := range hashes {
+		if remote, data, err := loadHash(client, h.url); err != nil {
+			if h.warn {
+				slog.Warn("Cannot fetch hash",
+					"hash", h.hashType,
+					"url", h.url,
+					"error", err)
+			} else {
+				slog.Info("Hash not present", "hash", h.hashType, "file", h.url)
+			}
+		} else {
+			switch h.hashType {
+			case algSha512:
+				{
+					remoteSha512 = remote
+					sha512Data = data
+				}
+			case algSha256:
+				{
+					remoteSha256 = remote
+					sha256Data = data
+				}
+			}
+			if h.preferred {
+				break
+			}
+		}
+	}
+	return remoteSha256, sha256Data, remoteSha512, sha512Data
+}
+
 func loadHash(client util.Client, p string) ([]byte, []byte, error) {
 	resp, err := client.Get(p)
 	if err != nil {