Merge unittest into sha-handling

commit 990c74a1a6
Merge: 86d7ce1 7824f3b
Author: koplas <pschwabauer@intevation.de>
Date:   Fri Nov 22 16:58:46 2024 +0100

    Merge branch 'sha-handling' into unittest

commit 86d7ce13dc
Merge: a6807d2 79b8900
Author: koplas <pschwabauer@intevation.de>
Date:   Fri Nov 22 16:54:45 2024 +0100

    Merge branch 'sha-handling' into unittest

commit 79b89009dd
Author: koplas <pschwabauer@intevation.de>
Date:   Fri Nov 22 16:31:56 2024 +0100

    Improve hash fetching and logging

commit a6807d24d6
Merge: ddb5518 d18d2c3
Author: koplas <pschwabauer@intevation.de>
Date:   Fri Nov 22 16:51:55 2024 +0100

    Merge branch 'sha-handling' into unittest

commit d18d2c3bf1
Author: koplas <pschwabauer@intevation.de>
Date:   Fri Nov 22 16:31:56 2024 +0100

    Improve hash fetching and logging

commit ddb5518c6d
Author: koplas <54645365+koplas@users.noreply.github.com>
Date:   Tue Sep 17 10:45:25 2024 +0200

    Extend SHA marking tests

commit 13c94f4fa0
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 20:46:31 2024 +0200

    Use temp directory for downloads

commit 1819b4896b
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 20:37:55 2024 +0200

    Fix rolie feed

commit 989e3667ba
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 20:23:22 2024 +0200

    Fix provider-metadata.json

commit 714735d74a
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 20:08:21 2024 +0200

    Implement provider handler

commit d488e39947
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 16:26:37 2024 +0200

    Add info about gpg key

commit a9bf9da130
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 16:12:49 2024 +0200

    Rename directory testdata

commit 6ca6dfee25
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 16:01:41 2024 +0200

    Add initial downloader tests

commit 20bee797c6
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 15:58:31 2024 +0200

    Fix: Remove unecessary error print

commit 8e4e508073
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 14:50:48 2024 +0200

    Extend links test

commit 3ba29f94de
Author: koplas <pschwabauer@intevation.de>
Date:   Mon Sep 16 14:11:14 2024 +0200

    Add initial directory feed testdata

commit dee55aafd9
Author: koplas <54645365+koplas@users.noreply.github.com>
Date:   Mon Sep 16 10:47:32 2024 +0200

    Add initial testdata

commit cd9338ae72
Author: koplas <54645365+koplas@users.noreply.github.com>
Date:   Thu Sep 12 15:54:42 2024 +0200

    Add initial download unittests

cmd/csaf_aggregator/client_test.go

@@ -0,0 +1,67 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
+
+package main
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/csaf-poc/csaf_distribution/v3/util"
+)
+
+func Test_downloadJSON(t *testing.T) {
+	tests := []struct {
+		name        string
+		statusCode  int
+		contentType string
+		wantErr     error
+	}{
+		{
+			name:        "status ok, application/json",
+			statusCode:  http.StatusOK,
+			contentType: "application/json",
+			wantErr:     nil,
+		},
+		{
+			name:        "status found, application/json",
+			statusCode:  http.StatusFound,
+			contentType: "application/json",
+			wantErr:     errNotFound,
+		},
+		{
+			name:        "status ok, application/xml",
+			statusCode:  http.StatusOK,
+			contentType: "application/xml",
+			wantErr:     errNotFound,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			found := func(r io.Reader) error {
+				return nil
+			}
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Add("Content-Type", test.contentType)
+				w.WriteHeader(test.statusCode)
+			}))
+			defer server.Close()
+			hClient := http.Client{}
+			client := util.Client(&hClient)
+			if gotErr := downloadJSON(client, server.URL, found); gotErr != test.wantErr {
+				t.Errorf("downloadJSON: Expected %q but got %q.", test.wantErr, gotErr)
+			}
+		})
+	}
+}

cmd/csaf_checker/links_test.go

@@ -10,8 +10,12 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/csaf-poc/csaf_distribution/v3/util"
 )
 
 const page0 = `<html>
@@ -31,7 +35,6 @@ const page0 = `<html>
 </html>`
 
 func TestLinksOnPage(t *testing.T) {
-
 	var links []string
 
 	err := linksOnPage(
@@ -58,3 +61,78 @@ func TestLinksOnPage(t *testing.T) {
 		}
 	}
 }
+
+func Test_listed(t *testing.T) {
+	tests := []struct {
+		name    string
+		badDirs util.Set[string]
+		path    string
+		want    bool
+	}{
+		{
+			name:    "listed path",
+			badDirs: util.Set[string]{},
+			path:    "/white/avendor-advisory-0004.json",
+			want:    true,
+		},
+		{
+			name:    "badDirs contains path",
+			badDirs: util.Set[string]{"/white/": {}},
+			path:    "/white/avendor-advisory-0004.json",
+			want:    false,
+		},
+		{
+			name:    "not found",
+			badDirs: util.Set[string]{},
+			path:    "/not-found/resource.json",
+			want:    false,
+		},
+		{
+			name:    "badDirs does not contain path",
+			badDirs: util.Set[string]{"/bad-dir/": {}},
+			path:    "/white/avendor-advisory-0004.json",
+			want:    true,
+		},
+		{
+			name:    "unlisted path",
+			badDirs: util.Set[string]{},
+			path:    "/white/avendor-advisory-0004-not-listed.json",
+			want:    false,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			serverURL := ""
+			fs := http.FileServer(http.Dir("../../testdata/simple-directory-provider"))
+			server := httptest.NewTLSServer(fs)
+			defer server.Close()
+
+			serverURL = server.URL
+
+			hClient := server.Client()
+			client := util.Client(hClient)
+
+			pgs := pages{}
+			cfg := config{RemoteValidator: "", RemoteValidatorCache: ""}
+			p, err := newProcessor(&cfg)
+			if err != nil {
+				t.Error(err)
+			}
+			p.client = client
+
+			badDirs := util.Set[string]{}
+			for dir := range test.badDirs {
+				badDirs.Add(serverURL + dir)
+			}
+
+			got, _ := pgs.listed(serverURL+test.path, p, badDirs)
+			if got != test.want {
+				t.Errorf("%q: Expected %t but got %t.", test.name, test.want, got)
+			}
+		})
+	}
+}

cmd/csaf_downloader/config.go

@@ -44,8 +44,8 @@ const (
 type hashAlgorithm string
 
 const (
-	algSha256 = hashAlgorithm("SHA256")
+	algSha256 = hashAlgorithm("sha256")
-	algSha512 = hashAlgorithm("SHA512")
+	algSha512 = hashAlgorithm("sha512")
 )
 
 type config struct {

cmd/csaf_downloader/downloader.go

@@ -47,6 +47,7 @@ type hashFetchInfo struct {
 
 type downloader struct {
 	cfg       *config
+	client    *util.Client // Used for testing
 	keys      *crypto.KeyRing
 	validator csaf.RemoteValidator
 	forwarder *forwarder
@@ -131,6 +132,11 @@ func (d *downloader) httpClient() util.Client {
 
 	client := util.Client(&hClient)
 
+	// Overwrite for testing purposes
+	if client != nil {
+		client = *d.client
+	}
+
 	// Add extra headers.
 	if len(d.cfg.ExtraHeader) > 0 {
 		client = &util.HeaderClient{

cmd/csaf_downloader/downloader_test.go

@@ -0,0 +1,218 @@
+// This file is Free Software under the Apache-2.0 License
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
+
+package main
+
+import (
+	"context"
+	"errors"
+	"html/template"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/csaf-poc/csaf_distribution/v3/util"
+)
+
+type ProviderParams struct {
+	URL          string
+	EnableSha256 bool
+	EnableSha512 bool
+}
+
+func ProviderHandler(params *ProviderParams, directoryProvider bool) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		path := "../../testdata/"
+		if directoryProvider {
+			path += "simple-directory-provider"
+		} else {
+			path += "simple-rolie-provider"
+		}
+
+		path += r.URL.Path
+
+		if strings.HasSuffix(r.URL.Path, "/") {
+			path += "index.html"
+		}
+
+		content, err := os.ReadFile(path)
+		if err != nil {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		switch {
+		case strings.HasSuffix(path, ".html"):
+			w.Header().Add("Content-Type", "text/html")
+		case strings.HasSuffix(path, ".json"):
+			w.Header().Add("Content-Type", "application/json")
+		case strings.HasSuffix(path, ".sha256") && directoryProvider && !params.EnableSha256:
+			w.WriteHeader(http.StatusNotFound)
+			return
+		case strings.HasSuffix(path, ".sha512") && directoryProvider && !params.EnableSha512:
+			w.WriteHeader(http.StatusNotFound)
+			return
+		default:
+			w.Header().Add("Content-Type", "text/plain")
+		}
+
+		tmplt, err := template.New("base").Parse(string(content))
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		err = tmplt.Execute(w, params)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+	})
+}
+
+func checkIfFileExists(path string, t *testing.T) bool {
+	if _, err := os.Stat(path); err == nil {
+		return true
+	} else if errors.Is(err, os.ErrNotExist) {
+		return false
+	} else {
+		t.Fatalf("Failed to check if file exists: %v", err)
+		return false
+	}
+}
+
+func TestShaMarking(t *testing.T) {
+	tests := []struct {
+		name              string
+		directoryProvider bool
+		wantSha256        bool
+		wantSha512        bool
+		enableSha256      bool
+		enableSha512      bool
+		preferredHash     hashAlgorithm
+	}{
+		{
+			name:              "want sha256 and sha512",
+			directoryProvider: false,
+			wantSha256:        true,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+		},
+		{
+			name:              "only want sha256",
+			directoryProvider: false,
+			wantSha256:        true,
+			wantSha512:        false,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha256,
+		},
+		{
+			name:              "only want sha512",
+			directoryProvider: false,
+			wantSha256:        false,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha512,
+		},
+		{
+			name:              "only want sha512",
+			directoryProvider: false,
+			wantSha256:        false,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha512,
+		},
+
+		{
+			name:              "only deliver sha256",
+			directoryProvider: false,
+			wantSha256:        true,
+			wantSha512:        false,
+			enableSha256:      true,
+			enableSha512:      false,
+			preferredHash:     algSha512,
+		},
+		{
+			name:              "only want sha256, directory provider",
+			directoryProvider: true,
+			wantSha256:        true,
+			wantSha512:        false,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha256,
+		},
+		{
+			name:              "only want sha512, directory provider",
+			directoryProvider: true,
+			wantSha256:        false,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha512,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			serverURL := ""
+			params := ProviderParams{
+				URL:          "",
+				EnableSha256: test.enableSha256,
+				EnableSha512: test.enableSha512,
+			}
+			server := httptest.NewTLSServer(ProviderHandler(&params, test.directoryProvider))
+			defer server.Close()
+
+			serverURL = server.URL
+			params.URL = server.URL
+
+			hClient := server.Client()
+			client := util.Client(hClient)
+
+			tempDir := t.TempDir()
+			cfg := config{LogLevel: &options.LogLevel{Level: slog.LevelDebug}, Directory: tempDir, PreferredHash: test.preferredHash}
+			err := cfg.prepare()
+			if err != nil {
+				t.Fatalf("SHA marking config failed: %v", err)
+			}
+			d, err := newDownloader(&cfg)
+			if err != nil {
+				t.Fatalf("could not init downloader: %v", err)
+			}
+			d.client = &client
+
+			ctx := context.Background()
+			err = d.run(ctx, []string{serverURL + "/provider-metadata.json"})
+			if err != nil {
+				t.Errorf("SHA marking %v: Expected no error, got: %v", test.name, err)
+			}
+			d.close()
+
+			// Check for downloaded hashes
+			sha256Exists := checkIfFileExists(tempDir+"/white/2020/avendor-advisory-0004.json.sha256", t)
+			sha512Exists := checkIfFileExists(tempDir+"/white/2020/avendor-advisory-0004.json.sha512", t)
+
+			if sha256Exists != test.wantSha256 {
+				t.Errorf("%v: expected sha256 hash present to be %v, got: %v", test.name, test.wantSha256, sha256Exists)
+			}
+
+			if sha512Exists != test.wantSha512 {
+				t.Errorf("%v: expected sha512 hash present to be %v, got: %v", test.name, test.wantSha512, sha512Exists)
+			}
+		})
+	}
+}

csaf/providermetaloader.go

@@ -352,7 +352,7 @@ func (pmdl *ProviderMetadataLoader) loadFromURL(path string) *LoadedProviderMeta
 	case len(errors) > 0:
 		result.Messages = []ProviderMetadataLoadMessage{{
 			Type:    SchemaValidationFailed,
-			Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err),
+			Message: fmt.Sprintf("%s: Validating against JSON schema failed", path),
 		}}
 		for _, msg := range errors {
 			result.Messages.Add(

testdata/simple-directory-provider/openpgp/info.txt

@@ -0,0 +1,2 @@
+The GPG key was generated with no passphrase and this command:
+`gpg --default-new-key-algo "ed25519/cert,sign+cv25519/encr" --quick-generate-key security@example.com"`

testdata/simple-directory-provider/openpgp/privkey.asc

@@ -0,0 +1,15 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lFgEZtsQNxYJKwYBBAHaRw8BAQdASr3y4zW+4XGqUlvRJ7stRCUHv8HB4ZoMoTtU
+KLgnHr4AAQD5G5xy/yTN5b+lvV5Ahrbz1qOZ/wmKTieGOH9GZb6JwhHwtBRzZWN1
+cml0eUBleGFtcGxlLmNvbYiZBBMWCgBBFiEEqJFMovEROcammgAY+zzZsV3mFZYF
+AmbbEDcCGwMFCQWjmoAFCwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AACgkQ+zzZ
+sV3mFZZskQEAg5Dttqm6TA7MtLxz7VSlklx95LQr9d5jm4jcOaqlGT0A/1mAAlUq
+SDySFGI6DFQLcaZaUd9Yl+1b0Icr0tUiOaQHnF0EZtsQNxIKKwYBBAGXVQEFAQEH
+QOTHP4FkopIGJMWXTYsaeQ1Dugd+yNYWB357vRYq6QsiAwEIBwAA/0RIazq1s8Oe
+23jvNaZGb/adDYnRrkCMXXTBKsuA6WOAEhKIeAQYFgoAIBYhBKiRTKLxETnGppoA
+GPs82bFd5hWWBQJm2xA3AhsMAAoJEPs82bFd5hWWDKABAOl+NoM6FBhKAvckUXDR
+MLZ4k778N4Vy9VHbectjRKj1AQCO3JOmON+U6/mjohXrc2bwzKzt2yGiLP2HMxDx
+uzMXBQ==
+=4XHC
+-----END PGP PRIVATE KEY BLOCK-----

testdata/simple-directory-provider/openpgp/pubkey.asc

@@ -0,0 +1,13 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mDMEZtsQNxYJKwYBBAHaRw8BAQdASr3y4zW+4XGqUlvRJ7stRCUHv8HB4ZoMoTtU
+KLgnHr60FHNlY3VyaXR5QGV4YW1wbGUuY29tiJkEExYKAEEWIQSokUyi8RE5xqaa
+ABj7PNmxXeYVlgUCZtsQNwIbAwUJBaOagAULCQgHAgIiAgYVCgkICwIEFgIDAQIe
+BwIXgAAKCRD7PNmxXeYVlmyRAQCDkO22qbpMDsy0vHPtVKWSXH3ktCv13mObiNw5
+qqUZPQD/WYACVSpIPJIUYjoMVAtxplpR31iX7VvQhyvS1SI5pAe4OARm2xA3Egor
+BgEEAZdVAQUBAQdA5Mc/gWSikgYkxZdNixp5DUO6B37I1hYHfnu9FirpCyIDAQgH
+iHgEGBYKACAWIQSokUyi8RE5xqaaABj7PNmxXeYVlgUCZtsQNwIbDAAKCRD7PNmx
+XeYVlgygAQDpfjaDOhQYSgL3JFFw0TC2eJO+/DeFcvVR23nLY0So9QEAjtyTpjjf
+lOv5o6IV63Nm8Mys7dshoiz9hzMQ8bszFwU=
+=rhGT
+-----END PGP PUBLIC KEY BLOCK-----

testdata/simple-directory-provider/provider-metadata.json

@@ -0,0 +1,25 @@
+{
+  "canonical_url": "{{.URL}}/provider-metadata.json",
+  "distributions": [
+    {
+      "directory_url": "{{.URL}}/white/"
+    }
+  ],
+  "last_updated": "2020-01-01T00:00:00Z",
+  "list_on_CSAF_aggregators": true,
+  "metadata_version": "2.0",
+  "mirror_on_CSAF_aggregators": true,
+  "public_openpgp_keys": [
+    {
+      "fingerprint": "A8914CA2F11139C6A69A0018FB3CD9B15DE61596",
+      "url": "{{.URL}}/openpgp/pubkey.asc"
+    }
+  ],
+  "publisher": {
+    "category": "vendor",
+    "name": "ACME Inc",
+    "namespace": "https://example.com",
+    "contact_details": "mailto:security@example.com"
+  },
+  "role": "csaf_trusted_provider"
+}

testdata/simple-directory-provider/security.txt

@@ -0,0 +1,2 @@
+CSAF: /provider-metadata.json
+

testdata/simple-directory-provider/white/avendor-advisory-0004-not-listed.json

@@ -0,0 +1,170 @@
+{
+  "document": {
+    "category": "csaf_vex",
+    "csaf_version": "2.0",
+    "distribution": {
+      "tlp": {
+        "label": "WHITE",
+        "url": "https://www.first.org/tlp/v1/"
+      }
+    },
+    "notes": [
+      {
+        "category": "summary",
+        "title": "Test document summary",
+        "text": "Auto generated test CSAF document"
+      }
+    ],
+    "publisher": {
+      "category": "vendor",
+      "name": "ACME Inc.",
+      "namespace": "https://www.example.com"
+    },
+    "title": "Test CSAF document",
+    "tracking": {
+      "current_release_date": "2020-01-01T00:00:00Z",
+      "generator": {
+        "date": "2020-01-01T00:00:00Z",
+        "engine": {
+          "name": "csaf-tool",
+          "version": "0.3.2"
+        }
+      },
+      "id": "Avendor-advisory-0004",
+      "initial_release_date": "2020-01-01T00:00:00Z",
+      "revision_history": [
+        {
+          "date": "2020-01-01T00:00:00Z",
+          "number": "1",
+          "summary": "Initial version"
+        }
+      ],
+      "status": "final",
+      "version": "1"
+    }
+  },
+  "product_tree": {
+    "branches": [
+      {
+        "category": "vendor",
+        "name": "AVendor",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_1",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "1.1",
+                "product": {
+                  "name": "AVendor product_1 1.1",
+                  "product_id": "CSAFPID_0001"
+                }
+              },
+              {
+                "category": "product_version",
+                "name": "1.2",
+                "product": {
+                  "name": "AVendor product_1 1.2",
+                  "product_id": "CSAFPID_0002"
+                }
+              },
+              {
+                "category": "product_version",
+                "name": "2.0",
+                "product": {
+                  "name": "AVendor product_1 2.0",
+                  "product_id": "CSAFPID_0003"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "category": "vendor",
+        "name": "AVendor1",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_2",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "1",
+                "product": {
+                  "name": "AVendor1 product_2 1",
+                  "product_id": "CSAFPID_0004"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "category": "vendor",
+        "name": "AVendor",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_3",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "2022H2",
+                "product": {
+                  "name": "AVendor product_3 2022H2",
+                  "product_id": "CSAFPID_0005"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "vulnerabilities": [
+    {
+      "cve": "CVE-2020-1234",
+      "notes": [
+        {
+          "category": "description",
+          "title": "CVE description",
+          "text": "https://nvd.nist.gov/vuln/detail/CVE-2020-1234"
+        }
+      ],
+      "product_status": {
+        "under_investigation": ["CSAFPID_0001"]
+      },
+      "threats": [
+        {
+          "category": "impact",
+          "details": "Customers should upgrade to the latest version of the product",
+          "date": "2020-01-01T00:00:00Z",
+          "product_ids": ["CSAFPID_0001"]
+        }
+      ]
+    },
+    {
+      "cve": "CVE-2020-9876",
+      "notes": [
+        {
+          "category": "description",
+          "title": "CVE description",
+          "text": "https://nvd.nist.gov/vuln/detail/CVE-2020-9876"
+        }
+      ],
+      "product_status": {
+        "under_investigation": ["CSAFPID_0001"]
+      },
+      "threats": [
+        {
+          "category": "impact",
+          "details": "Still under investigation",
+          "date": "2020-01-01T00:00:00Z",
+          "product_ids": ["CSAFPID_0001"]
+        }
+      ]
+    }
+  ]
+}

testdata/simple-directory-provider/white/avendor-advisory-0004.json

@@ -0,0 +1,170 @@
+{
+  "document": {
+    "category": "csaf_vex",
+    "csaf_version": "2.0",
+    "distribution": {
+      "tlp": {
+        "label": "WHITE",
+        "url": "https://www.first.org/tlp/v1/"
+      }
+    },
+    "notes": [
+      {
+        "category": "summary",
+        "title": "Test document summary",
+        "text": "Auto generated test CSAF document"
+      }
+    ],
+    "publisher": {
+      "category": "vendor",
+      "name": "ACME Inc.",
+      "namespace": "https://www.example.com"
+    },
+    "title": "Test CSAF document",
+    "tracking": {
+      "current_release_date": "2020-01-01T00:00:00Z",
+      "generator": {
+        "date": "2020-01-01T00:00:00Z",
+        "engine": {
+          "name": "csaf-tool",
+          "version": "0.3.2"
+        }
+      },
+      "id": "Avendor-advisory-0004",
+      "initial_release_date": "2020-01-01T00:00:00Z",
+      "revision_history": [
+        {
+          "date": "2020-01-01T00:00:00Z",
+          "number": "1",
+          "summary": "Initial version"
+        }
+      ],
+      "status": "final",
+      "version": "1"
+    }
+  },
+  "product_tree": {
+    "branches": [
+      {
+        "category": "vendor",
+        "name": "AVendor",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_1",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "1.1",
+                "product": {
+                  "name": "AVendor product_1 1.1",
+                  "product_id": "CSAFPID_0001"
+                }
+              },
+              {
+                "category": "product_version",
+                "name": "1.2",
+                "product": {
+                  "name": "AVendor product_1 1.2",
+                  "product_id": "CSAFPID_0002"
+                }
+              },
+              {
+                "category": "product_version",
+                "name": "2.0",
+                "product": {
+                  "name": "AVendor product_1 2.0",
+                  "product_id": "CSAFPID_0003"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "category": "vendor",
+        "name": "AVendor1",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_2",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "1",
+                "product": {
+                  "name": "AVendor1 product_2 1",
+                  "product_id": "CSAFPID_0004"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "category": "vendor",
+        "name": "AVendor",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_3",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "2022H2",
+                "product": {
+                  "name": "AVendor product_3 2022H2",
+                  "product_id": "CSAFPID_0005"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "vulnerabilities": [
+    {
+      "cve": "CVE-2020-1234",
+      "notes": [
+        {
+          "category": "description",
+          "title": "CVE description",
+          "text": "https://nvd.nist.gov/vuln/detail/CVE-2020-1234"
+        }
+      ],
+      "product_status": {
+        "under_investigation": ["CSAFPID_0001"]
+      },
+      "threats": [
+        {
+          "category": "impact",
+          "details": "Customers should upgrade to the latest version of the product",
+          "date": "2020-01-01T00:00:00Z",
+          "product_ids": ["CSAFPID_0001"]
+        }
+      ]
+    },
+    {
+      "cve": "CVE-2020-9876",
+      "notes": [
+        {
+          "category": "description",
+          "title": "CVE description",
+          "text": "https://nvd.nist.gov/vuln/detail/CVE-2020-9876"
+        }
+      ],
+      "product_status": {
+        "under_investigation": ["CSAFPID_0001"]
+      },
+      "threats": [
+        {
+          "category": "impact",
+          "details": "Still under investigation",
+          "date": "2020-01-01T00:00:00Z",
+          "product_ids": ["CSAFPID_0001"]
+        }
+      ]
+    }
+  ]
+}

testdata/simple-directory-provider/white/avendor-advisory-0004.json.asc

@@ -0,0 +1,7 @@
+-----BEGIN PGP SIGNATURE-----
+
+iHUEABYKAB0WIQSokUyi8RE5xqaaABj7PNmxXeYVlgUCZukv9QAKCRD7PNmxXeYV
+ljq0AP9n/rTgoNCJzSTZzNrrMy28ZR+Ppp1MSPWGFUzsx6qLJgD/d8cu0lokMsXf
+y0uc9k7hrla/ajFUzNt3AVvT+CPFtAo=
+=7E66
+-----END PGP SIGNATURE-----

testdata/simple-directory-provider/white/avendor-advisory-0004.json.sha256

@@ -0,0 +1 @@
+cb263bf1beab18b893de63f2966d0d8c5f38d60101c24d3fd7a5feebaad02c3b  avendor-advisory-0004.json

testdata/simple-directory-provider/white/avendor-advisory-0004.json.sha512

@@ -0,0 +1 @@
+39476e1d08a0871d166091c90de259544382a3599eebda118a93468499a30fd034286086c461a97d3d5298e093b0be3868e8d89d8a6a255c4aa6adb81ebbfcad  avendor-advisory-0004.json

testdata/simple-directory-provider/white/changes.csv

@@ -0,0 +1 @@
+"avendor-advisory-0004.json","2020-01-01T00:00:00+00:00"

testdata/simple-directory-provider/white/index.html

@@ -0,0 +1,6 @@
+<!doctype html>
+<html>
+  <body>
+    <a href="./avendor-advisory-0004.json">avendor-advisory-0004</a>
+  </body>
+</html>

testdata/simple-directory-provider/white/index.txt

@@ -0,0 +1 @@
+avendor-advisory-0004.json

testdata/simple-rolie-provider/openpgp/info.txt

@@ -0,0 +1,2 @@
+The GPG key was generated with no passphrase and this command:
+`gpg --default-new-key-algo "ed25519/cert,sign+cv25519/encr" --quick-generate-key security@example.com"`

testdata/simple-rolie-provider/openpgp/privkey.asc

@@ -0,0 +1,15 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lFgEZtsQNxYJKwYBBAHaRw8BAQdASr3y4zW+4XGqUlvRJ7stRCUHv8HB4ZoMoTtU
+KLgnHr4AAQD5G5xy/yTN5b+lvV5Ahrbz1qOZ/wmKTieGOH9GZb6JwhHwtBRzZWN1
+cml0eUBleGFtcGxlLmNvbYiZBBMWCgBBFiEEqJFMovEROcammgAY+zzZsV3mFZYF
+AmbbEDcCGwMFCQWjmoAFCwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AACgkQ+zzZ
+sV3mFZZskQEAg5Dttqm6TA7MtLxz7VSlklx95LQr9d5jm4jcOaqlGT0A/1mAAlUq
+SDySFGI6DFQLcaZaUd9Yl+1b0Icr0tUiOaQHnF0EZtsQNxIKKwYBBAGXVQEFAQEH
+QOTHP4FkopIGJMWXTYsaeQ1Dugd+yNYWB357vRYq6QsiAwEIBwAA/0RIazq1s8Oe
+23jvNaZGb/adDYnRrkCMXXTBKsuA6WOAEhKIeAQYFgoAIBYhBKiRTKLxETnGppoA
+GPs82bFd5hWWBQJm2xA3AhsMAAoJEPs82bFd5hWWDKABAOl+NoM6FBhKAvckUXDR
+MLZ4k778N4Vy9VHbectjRKj1AQCO3JOmON+U6/mjohXrc2bwzKzt2yGiLP2HMxDx
+uzMXBQ==
+=4XHC
+-----END PGP PRIVATE KEY BLOCK-----

testdata/simple-rolie-provider/openpgp/pubkey.asc

@@ -0,0 +1,13 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mDMEZtsQNxYJKwYBBAHaRw8BAQdASr3y4zW+4XGqUlvRJ7stRCUHv8HB4ZoMoTtU
+KLgnHr60FHNlY3VyaXR5QGV4YW1wbGUuY29tiJkEExYKAEEWIQSokUyi8RE5xqaa
+ABj7PNmxXeYVlgUCZtsQNwIbAwUJBaOagAULCQgHAgIiAgYVCgkICwIEFgIDAQIe
+BwIXgAAKCRD7PNmxXeYVlmyRAQCDkO22qbpMDsy0vHPtVKWSXH3ktCv13mObiNw5
+qqUZPQD/WYACVSpIPJIUYjoMVAtxplpR31iX7VvQhyvS1SI5pAe4OARm2xA3Egor
+BgEEAZdVAQUBAQdA5Mc/gWSikgYkxZdNixp5DUO6B37I1hYHfnu9FirpCyIDAQgH
+iHgEGBYKACAWIQSokUyi8RE5xqaaABj7PNmxXeYVlgUCZtsQNwIbDAAKCRD7PNmx
+XeYVlgygAQDpfjaDOhQYSgL3JFFw0TC2eJO+/DeFcvVR23nLY0So9QEAjtyTpjjf
+lOv5o6IV63Nm8Mys7dshoiz9hzMQ8bszFwU=
+=rhGT
+-----END PGP PUBLIC KEY BLOCK-----

testdata/simple-rolie-provider/provider-metadata.json

@@ -0,0 +1,33 @@
+{
+  "canonical_url": "{{.URL}}/provider-metadata.json",
+  "distributions": [
+    {
+      "rolie": {
+        "feeds": [
+          {
+            "summary": "TLP:WHITE advisories",
+            "tlp_label": "WHITE",
+            "url": "{{.URL}}/white/white-feed.json"
+          }
+        ]
+      }
+    }
+  ],
+  "last_updated": "2020-01-01T00:00:00Z",
+  "list_on_CSAF_aggregators": true,
+  "metadata_version": "2.0",
+  "mirror_on_CSAF_aggregators": true,
+  "public_openpgp_keys": [
+    {
+      "fingerprint": "A8914CA2F11139C6A69A0018FB3CD9B15DE61596",
+      "url": "{{.URL}}/openpgp/pubkey.asc"
+    }
+  ],
+  "publisher": {
+    "category": "vendor",
+    "name": "ACME Inc",
+    "namespace": "https://example.com",
+    "contact_details": "mailto:security@example.com"
+  },
+  "role": "csaf_trusted_provider"
+}

testdata/simple-rolie-provider/security.txt

@@ -0,0 +1,2 @@
+CSAF: /provider-metadata.json
+

testdata/simple-rolie-provider/service.json

@@ -0,0 +1,23 @@
+{
+  "service": {
+    "workspace": [
+      {
+        "title": "CSAF feeds",
+        "collection": [
+          {
+            "title": "CSAF feed (TLP:WHITE)",
+            "href": "/white/white-feed.json",
+            "categories": {
+              "category": [
+                {
+                  "scheme": "urn:ietf:params:rolie:category:information-type",
+                  "term": "csaf"
+                }
+              ]
+            }
+          }
+        ]
+      }
+    ]
+  }
+}

testdata/simple-rolie-provider/white/avendor-advisory-0004.json

@@ -0,0 +1,170 @@
+{
+  "document": {
+    "category": "csaf_vex",
+    "csaf_version": "2.0",
+    "distribution": {
+      "tlp": {
+        "label": "WHITE",
+        "url": "https://www.first.org/tlp/v1/"
+      }
+    },
+    "notes": [
+      {
+        "category": "summary",
+        "title": "Test document summary",
+        "text": "Auto generated test CSAF document"
+      }
+    ],
+    "publisher": {
+      "category": "vendor",
+      "name": "ACME Inc.",
+      "namespace": "https://www.example.com"
+    },
+    "title": "Test CSAF document",
+    "tracking": {
+      "current_release_date": "2020-01-01T00:00:00Z",
+      "generator": {
+        "date": "2020-01-01T00:00:00Z",
+        "engine": {
+          "name": "csaf-tool",
+          "version": "0.3.2"
+        }
+      },
+      "id": "Avendor-advisory-0004",
+      "initial_release_date": "2020-01-01T00:00:00Z",
+      "revision_history": [
+        {
+          "date": "2020-01-01T00:00:00Z",
+          "number": "1",
+          "summary": "Initial version"
+        }
+      ],
+      "status": "final",
+      "version": "1"
+    }
+  },
+  "product_tree": {
+    "branches": [
+      {
+        "category": "vendor",
+        "name": "AVendor",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_1",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "1.1",
+                "product": {
+                  "name": "AVendor product_1 1.1",
+                  "product_id": "CSAFPID_0001"
+                }
+              },
+              {
+                "category": "product_version",
+                "name": "1.2",
+                "product": {
+                  "name": "AVendor product_1 1.2",
+                  "product_id": "CSAFPID_0002"
+                }
+              },
+              {
+                "category": "product_version",
+                "name": "2.0",
+                "product": {
+                  "name": "AVendor product_1 2.0",
+                  "product_id": "CSAFPID_0003"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "category": "vendor",
+        "name": "AVendor1",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_2",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "1",
+                "product": {
+                  "name": "AVendor1 product_2 1",
+                  "product_id": "CSAFPID_0004"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "category": "vendor",
+        "name": "AVendor",
+        "branches": [
+          {
+            "category": "product_name",
+            "name": "product_3",
+            "branches": [
+              {
+                "category": "product_version",
+                "name": "2022H2",
+                "product": {
+                  "name": "AVendor product_3 2022H2",
+                  "product_id": "CSAFPID_0005"
+                }
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "vulnerabilities": [
+    {
+      "cve": "CVE-2020-1234",
+      "notes": [
+        {
+          "category": "description",
+          "title": "CVE description",
+          "text": "https://nvd.nist.gov/vuln/detail/CVE-2020-1234"
+        }
+      ],
+      "product_status": {
+        "under_investigation": ["CSAFPID_0001"]
+      },
+      "threats": [
+        {
+          "category": "impact",
+          "details": "Customers should upgrade to the latest version of the product",
+          "date": "2020-01-01T00:00:00Z",
+          "product_ids": ["CSAFPID_0001"]
+        }
+      ]
+    },
+    {
+      "cve": "CVE-2020-9876",
+      "notes": [
+        {
+          "category": "description",
+          "title": "CVE description",
+          "text": "https://nvd.nist.gov/vuln/detail/CVE-2020-9876"
+        }
+      ],
+      "product_status": {
+        "under_investigation": ["CSAFPID_0001"]
+      },
+      "threats": [
+        {
+          "category": "impact",
+          "details": "Still under investigation",
+          "date": "2020-01-01T00:00:00Z",
+          "product_ids": ["CSAFPID_0001"]
+        }
+      ]
+    }
+  ]
+}

testdata/simple-rolie-provider/white/avendor-advisory-0004.json.asc

@@ -0,0 +1,7 @@
+-----BEGIN PGP SIGNATURE-----
+
+iHUEABYKAB0WIQSokUyi8RE5xqaaABj7PNmxXeYVlgUCZukv9QAKCRD7PNmxXeYV
+ljq0AP9n/rTgoNCJzSTZzNrrMy28ZR+Ppp1MSPWGFUzsx6qLJgD/d8cu0lokMsXf
+y0uc9k7hrla/ajFUzNt3AVvT+CPFtAo=
+=7E66
+-----END PGP SIGNATURE-----

testdata/simple-rolie-provider/white/avendor-advisory-0004.json.sha256

@@ -0,0 +1 @@
+cb263bf1beab18b893de63f2966d0d8c5f38d60101c24d3fd7a5feebaad02c3b  avendor-advisory-0004.json

testdata/simple-rolie-provider/white/avendor-advisory-0004.json.sha512

@@ -0,0 +1 @@
+39476e1d08a0871d166091c90de259544382a3599eebda118a93468499a30fd034286086c461a97d3d5298e093b0be3868e8d89d8a6a255c4aa6adb81ebbfcad  avendor-advisory-0004.json

testdata/simple-rolie-provider/white/white-feed.json

@@ -0,0 +1,61 @@
+{
+  "feed": {
+    "id": "csaf-feed-tlp-white",
+    "title": "CSAF feed (TLP:WHITE)",
+    "link": [
+      {
+        "rel": "self",
+        "href": "/white/csaf-feed-tlp-white.json"
+      },
+      {
+        "rel": "service",
+        "href": "/service.json"
+      }
+    ],
+    "category": [
+      {
+        "scheme": "urn:ietf:params:rolie:category:information-type",
+        "term": "csaf"
+      }
+    ],
+    "updated": "2020-01-01T00:00:00Z",
+    "entry": [
+      {
+        "id": "Avendor-advisory-0004",
+        "title": "Test CSAF document",
+        "link": [
+          {
+            "rel": "self",
+            "href": "/white/avendor-advisory-0004.json"
+          },
+          {{if .EnableSha256}}
+          {
+            "rel": "hash",
+            "href": "/white/avendor-advisory-0004.json.sha256"
+          },
+          {{end}}
+          {{if .EnableSha512}}
+          {
+            "rel": "hash",
+            "href": "/white/avendor-advisory-0004.json.sha512"
+          },
+          {{end}}
+          {
+            "rel": "signature",
+            "href": "/white/avendor-advisory-0004.json.asc"
+          }
+        ],
+        "published": "2020-01-01T00:00:00Z",
+        "updated": "2020-01-01T00:00:00Z",
+        "content": {
+          "type": "application/json",
+          "src": "/avendor-advisory-0004.json"
+        },
+        "format": {
+          "schema": "https://docs.oasis-open.org/csaf/csaf/v2.0/csaf_json_schema.json",
+          "version": "2.0"
+        }
+      }
+    ]
+  }
+}