Improve SHA* marking

cmd/csaf_checker/processor.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"path/filepath"
@@ -138,7 +139,7 @@ func (m *topicMessages) info(format string, args ...any) {
 	m.add(InfoType, format, args...)
 }
 
-// use signals that we going to use this topic.
+// use signals that we're going to use this topic.
 func (m *topicMessages) use() {
 	if *m == nil {
 		*m = []Message{}
@@ -164,7 +165,7 @@ func (m *topicMessages) hasErrors() bool {
 	return false
 }
 
-// newProcessor returns an initilaized processor.
+// newProcessor returns an initialized processor.
 func newProcessor(cfg *config) (*processor, error) {
 
 	var validator csaf.RemoteValidator
@@ -594,10 +595,15 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
 
 		var file csaf.AdvisoryFile
 
-		if sha256 != "" || sha512 != "" || sign != "" {
+		switch {
-			file = csaf.HashedAdvisoryFile{url, sha256, sha512, sign}
+		case sha256 == "" && sha512 == "":
-		} else {
+			slog.Error("No hash listed on ROLIE feed", "file", url)
-			file = csaf.PlainAdvisoryFile(url)
+			return
+		case sign == "":
+			slog.Error("No signature listed on ROLIE feed", "file", url)
+			return
+		default:
+			file = csaf.PlainAdvisoryFile{Path: url, SHA256: sha256, SHA512: sha512, Sign: sign}
 		}
 
 		files = append(files, file)
@@ -888,7 +894,16 @@ func (p *processor) checkIndex(base string, mask whereType) error {
 				p.badIntegrities.error("index.txt contains invalid URL %q in line %d", u, line)
 				continue
 			}
-			files = append(files, csaf.PlainAdvisoryFile(u))
+
+			SHA256 := p.checkURL(u + ".sha256")
+			SHA512 := p.checkURL(u + ".sha512")
+			sign := p.checkURL(u + ".asc")
+			files = append(files, csaf.PlainAdvisoryFile{
+				Path:   u,
+				SHA256: SHA256,
+				SHA512: SHA512,
+				Sign:   sign,
+			})
 		}
 		return files, scanner.Err()
 	}()
@@ -906,6 +921,15 @@ func (p *processor) checkIndex(base string, mask whereType) error {
 	return p.integrity(files, base, mask, p.badIndices.add)
 }
 
+// checkURL returns the URL if it is accessible.
+func (p *processor) checkURL(url string) string {
+	_, err := p.client.Head(url)
+	if err != nil {
+		return url
+	}
+	return ""
+}
+
 // checkChanges fetches the "changes.csv" and calls the "checkTLS" method for HTTPs checks.
 // It extracts the file content, tests the column number and the validity of the time format
 // of the fields' values and if they are sorted properly. Then it passes the files to the
@@ -970,9 +994,14 @@ func (p *processor) checkChanges(base string, mask whereType) error {
 				continue
 			}
 			path := r[pathColumn]
+
+			SHA256 := p.checkURL(path + ".sha256")
+			SHA512 := p.checkURL(path + ".sha512")
+			sign := p.checkURL(path + ".asc")
+
 			times, files =
 				append(times, t),
-				append(files, csaf.PlainAdvisoryFile(path))
+				append(files, csaf.PlainAdvisoryFile{Path: path, SHA256: SHA256, SHA512: SHA512, Sign: sign})
 		}
 		return times, files, nil
 	}()

cmd/csaf_downloader/downloader.go

@@ -501,7 +501,10 @@ nextAdvisory:
 			signData                   []byte
 		)
 
-		// Only hash when we have a remote counter part we can compare it with.
+		if file.SHA256URL() == "" {
+			slog.Info("SHA256 not present", "file", file.URL())
+		} else {
+			// Only hash when we have a remote counterpart we can compare it with.
 			if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
 				slog.Warn("Cannot fetch SHA256",
 					"url", file.SHA256URL(),
@@ -510,7 +513,11 @@ nextAdvisory:
 				s256 = sha256.New()
 				writers = append(writers, s256)
 			}
+		}
 
+		if file.SHA512URL() == "" {
+			slog.Info("SHA512 not present", "file", file.URL())
+		} else {
 			if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
 				slog.Warn("Cannot fetch SHA512",
 					"url", file.SHA512URL(),
@@ -519,6 +526,7 @@ nextAdvisory:
 				s512 = sha512.New()
 				writers = append(writers, s512)
 			}
+		}
 
 		// Remember the data as we need to store it to file later.
 		data.Reset()

csaf/advisories.go

@@ -34,55 +34,30 @@ type AdvisoryFile interface {
 // PlainAdvisoryFile is a simple implementation of checkFile.
 // The hash and signature files are directly constructed by extending
 // the file name.
-type PlainAdvisoryFile string
+type PlainAdvisoryFile struct {
+	Path   string
+	SHA256 string
+	SHA512 string
+	Sign   string
+}
 
 // URL returns the URL of this advisory.
-func (paf PlainAdvisoryFile) URL() string { return string(paf) }
+func (paf PlainAdvisoryFile) URL() string { return paf.Path }
 
 // SHA256URL returns the URL of SHA256 hash file of this advisory.
-func (paf PlainAdvisoryFile) SHA256URL() string { return string(paf) + ".sha256" }
+func (paf PlainAdvisoryFile) SHA256URL() string { return paf.SHA256 }
 
 // SHA512URL returns the URL of SHA512 hash file of this advisory.
-func (paf PlainAdvisoryFile) SHA512URL() string { return string(paf) + ".sha512" }
+func (paf PlainAdvisoryFile) SHA512URL() string { return paf.SHA512 }
 
 // SignURL returns the URL of signature file of this advisory.
-func (paf PlainAdvisoryFile) SignURL() string { return string(paf) + ".asc" }
+func (paf PlainAdvisoryFile) SignURL() string { return paf.Sign }
 
 // LogValue implements [slog.LogValuer]
 func (paf PlainAdvisoryFile) LogValue() slog.Value {
 	return slog.GroupValue(slog.String("url", paf.URL()))
 }
 
-// HashedAdvisoryFile is a more involed version of checkFile.
-// Here each component can be given explicitly.
-// If a component is not given it is constructed by
-// extending the first component.
-type HashedAdvisoryFile [4]string
-
-func (haf HashedAdvisoryFile) name(i int, ext string) string {
-	if haf[i] != "" {
-		return haf[i]
-	}
-	return haf[0] + ext
-}
-
-// URL returns the URL of this advisory.
-func (haf HashedAdvisoryFile) URL() string { return haf[0] }
-
-// SHA256URL returns the URL of SHA256 hash file of this advisory.
-func (haf HashedAdvisoryFile) SHA256URL() string { return haf.name(1, ".sha256") }
-
-// SHA512URL returns the URL of SHA512 hash file of this advisory.
-func (haf HashedAdvisoryFile) SHA512URL() string { return haf.name(2, ".sha512") }
-
-// SignURL returns the URL of signature file of this advisory.
-func (haf HashedAdvisoryFile) SignURL() string { return haf.name(3, ".asc") }
-
-// LogValue implements [slog.LogValuer]
-func (haf HashedAdvisoryFile) LogValue() slog.Value {
-	return slog.GroupValue(slog.String("url", haf.URL()))
-}
-
 // AdvisoryFileProcessor implements the extraction of
 // advisory file names from a given provider metadata.
 type AdvisoryFileProcessor struct {
@@ -120,7 +95,7 @@ func empty(arr []string) bool {
 	return true
 }
 
-// Process extracts the adivisory filenames and passes them with
+// Process extracts the advisory filenames and passes them with
 // the corresponding label to fn.
 func (afp *AdvisoryFileProcessor) Process(
 	fn func(TLPLabel, []AdvisoryFile) error,
@@ -201,6 +176,15 @@ func (afp *AdvisoryFileProcessor) Process(
 	return nil
 }
 
+// checkURL returns the URL if it is accessible.
+func (afp *AdvisoryFileProcessor) checkURL(url string) string {
+	_, err := afp.client.Head(url)
+	if err != nil {
+		return url
+	}
+	return ""
+}
+
 // loadChanges loads baseURL/changes.csv and returns a list of files
 // prefixed by baseURL/.
 func (afp *AdvisoryFileProcessor) loadChanges(
@@ -257,8 +241,19 @@ func (afp *AdvisoryFileProcessor) loadChanges(
 			lg("%q contains an invalid URL %q in line %d", changesURL, path, line)
 			continue
 		}
+
+		self := base.JoinPath(path).String()
+		sha256 := afp.checkURL(self + ".sha256")
+		sha512 := afp.checkURL(self + ".sha512")
+		sign := afp.checkURL(self + ".asc")
+
 		files = append(files,
-			PlainAdvisoryFile(base.JoinPath(path).String()))
+			PlainAdvisoryFile{
+				Path:   path,
+				SHA256: sha256,
+				SHA512: sha512,
+				Sign:   sign,
+			})
 	}
 	return files, nil
 }
@@ -325,7 +320,6 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 		}
 
 		rfeed.Entries(func(entry *Entry) {
-
 			// Filter if we have date checking.
 			if afp.AgeAccept != nil {
 				if t := time.Time(entry.Updated); !t.IsZero() && !afp.AgeAccept(t) {
@@ -359,10 +353,15 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 
 			var file AdvisoryFile
 
-			if sha256 != "" || sha512 != "" || sign != "" {
+			switch {
-				file = HashedAdvisoryFile{self, sha256, sha512, sign}
+			case sha256 == "" && sha512 == "":
-			} else {
+				slog.Error("No hash listed on ROLIE feed", "file", self)
-				file = PlainAdvisoryFile(self)
+				return
+			case sign == "":
+				slog.Error("No signature listed on ROLIE feed", "file", self)
+				return
+			default:
+				file = PlainAdvisoryFile{self, sha256, sha512, sign}
 			}
 
 			files = append(files, file)