Update 3rd party libraries. (#711)

ROLIE: Add category to entries

.github/workflows/generate-markdown.yml

@@ -13,8 +13,8 @@ jobs:
   auto-update-readme:
       runs-on: ubuntu-latest
       steps:
-        - uses: actions/checkout@v2
+        - uses: actions/checkout@v5
         - name: Markdown autodocs
-          uses: dineshsonachalam/markdown-autodocs@v1.0.4
+          uses: dineshsonachalam/markdown-autodocs@v1.0.7
           with:
             output_file_paths: '[./README.md, ./docs/*.md]'

.github/workflows/go-oldstable.yml

@@ -0,0 +1,26 @@
+name: Go Test (oldstable)
+
+on:
+  push:
+    paths:
+      - "**.go"
+  pull_request:
+    paths:
+      - "**.go"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: 'oldstable'
+
+      - name: Build
+        run: go build -v ./cmd/...
+
+      - name: Tests
+        run: go test -v ./...

.github/workflows/go.yml

@@ -4,17 +4,21 @@ on:
   push:
     paths:
       - "**.go"
+  pull_request:
+    paths:
+      - "**.go"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v6
         with:
-          go-version: 1.21.0
+          go-version: "stable"
 
       - name: Build
         run: go build -v ./cmd/...
@@ -23,15 +27,36 @@ jobs:
         run: go vet ./...
 
       - name: gofmt
-        uses: Jerome1337/gofmt-action@v1.0.4
+        uses: Jerome1337/gofmt-action@v1.0.5
         with:
           gofmt-flags: "-l -d"
 
-      - name: golint
-        uses: Jerome1337/golint-action@v1.0.2
-
       - name: Revive Action
-        uses: morphy2k/revive-action@v2.5.1
+        uses: morphy2k/revive-action@v2
 
       - name: Tests
         run: go test -v ./...
+
+  run_modver:
+      runs-on: ubuntu-latest
+      needs: build # Only run when build job was successful
+      if: ${{ github.event_name == 'pull_request' && success() }}
+      permissions:
+        contents: read # Modver needs to read the repo content
+        pull-requests: write # Modver needs to write comments/status on PRs
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v5
+          with:
+            fetch-depth: 0 # Modver needs full history for comparison
+
+        - name: Set up Go
+          uses: actions/setup-go@v6
+          with:
+            go-version: "stable"
+
+        - name: Modver
+          uses: bobg/modver@v2.12.0
+          with:
+            github_token: ${{ secrets.GITHUB_TOKEN }}
+            pull_request_url: https://github.com/${{ github.repository }}/pull/${{ github.event.number }}

.github/workflows/itest.yml

@@ -5,19 +5,19 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
-          go-version: 1.21.0
+          go-version-file: "go.mod"
+          check-latest: true
 
       - name: Set up Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v6
         with:
-          node-version: 16
+          node-version: 24
-
-      - name: Checkout
-        uses: actions/checkout@v3
 
       - name: Execute the scripts
         run: |
@@ -25,7 +25,7 @@ jobs:
           sudo apt install -y make nginx fcgiwrap gnutls-bin
           cp -r $GITHUB_WORKSPACE ~
           cd ~
-          cd csaf_distribution/docs/scripts/
+          cd csaf/docs/scripts/
           # keep in sync with docs/scripts/Readme.md
           export FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)"
           source ./TLSConfigsForITest.sh
@@ -36,10 +36,10 @@ jobs:
         shell: bash
 
       - name: Upload test results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-            name: checker-results
+          name: checker-results
-            path: |
+          path: |
-                ~/checker-results.html
+            ~/checker-results.html
-                ~/checker-results-no-clientcert.json
+            ~/checker-results-no-clientcert.json
-            if-no-files-found: error
+          if-no-files-found: error

.github/workflows/release.yml

@@ -7,22 +7,26 @@ on:
 jobs:
   releases-matrix:
     name: Release Go binaries
-    runs-on: ubuntu-20.04
+    # use oldest available ubuntu to be compatible with more libc.so revs.
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
-            go-version: '^1.21.0'
+            go-version: '^1.24.9'
+            check-latest: true
 
       - name: Build
         run: make dist
 
       - name: Upload release assets
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
             files: |
-                dist/csaf_distribution-*.zip
+                dist/csaf-*.zip
-                dist/csaf_distribution-*.tar.gz
+                dist/csaf-*.tar.gz

LICENSE-Apache-2.0.txt

@@ -0,0 +1,73 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+     (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+
+     (b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+
+     (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+
+     (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+
+     You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

LICENSES/Apache-2.0.txt

@@ -0,0 +1,73 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+     (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+
+     (b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+
+     (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+
+     (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+
+     You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

LICENSES/BSD-3-Clause.txt

@@ -0,0 +1,11 @@
+Copyright (c) <year> <owner>. 
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSES/LicenseRef-Go119-BSD-Patentgrant.txt

@@ -1,51 +0,0 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-Additional IP Rights Grant (Patents)
-
-"This implementation" means the copyrightable works distributed by
-Google as part of the Go project.
-
-Google hereby grants to You a perpetual, worldwide, non-exclusive,
-no-charge, royalty-free, irrevocable (except as stated in this section)
-patent license to make, have made, use, offer to sell, sell, import,
-transfer and otherwise run, modify and propagate the contents of this
-implementation of Go, where such license applies only to those patent
-claims, both currently owned or controlled by Google and acquired in
-the future, licensable by Google that are necessarily infringed by this
-implementation of Go.  This grant does not include claims that would be
-infringed only as a consequence of further modification of this
-implementation.  If you or your agent or exclusive licensee institute or
-order or agree to the institution of patent litigation against any
-entity (including a cross-claim or counterclaim in a lawsuit) alleging
-that this implementation of Go or any code incorporated within this
-implementation of Go constitutes direct or contributory patent
-infringement, or inducement of patent infringement, then any patent
-rights granted to you under this License for this implementation of Go
-shall terminate as of the date such litigation is filed.

Makefile

@@ -1,26 +1,26 @@
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
 #
-# Makefile to build csaf_distribution components
+# Makefile to build csaf components
 
 SHELL = /bin/bash
 BUILD = go build
 MKDIR = mkdir -p
 
-.PHONY: build build_linux build_win build_mac_amd64 build_mac_arm64 tag_checked_out mostlyclean
+.PHONY: build build_linux build_linux_arm64 build_win build_win_arm64 build_mac_amd64 build_mac_arm64 tag_checked_out mostlyclean
 
 all:
-	@echo choose a target from: build build_linux build_win build_mac_amd64 build_mac_arm64 mostlyclean
+	@echo choose a target from: build build_linux build_linux_arm64 build_win build_win_arm64 build_mac_amd64 build_mac_arm64 mostlyclean
 	@echo prepend \`make BUILDTAG=1\` to checkout the highest git tag before building
 	@echo or set BUILDTAG to a specific tag
 
 # Build all binaries
-build: build_linux build_win build_mac_amd64 build_mac_arm64
+build: build_linux build_linux_arm64 build_win build_win_arm64 build_mac_amd64 build_mac_arm64
 
 # if BUILDTAG == 1 set it to the highest git tag
 ifeq ($(strip $(BUILDTAG)),1)
@@ -29,7 +29,7 @@ endif
 
 ifdef BUILDTAG
 # add the git tag checkout to the requirements of our build targets
-build_linux build_win build_mac_amd64 build_mac_arm64: tag_checked_out
+build_linux build_linux_arm64 build_win build_win_arm64 build_mac_amd64 build_mac_arm64: tag_checked_out
 endif
 
 tag_checked_out:
@@ -41,54 +41,77 @@ tag_checked_out:
 # into a semver version. For this we increase the PATCH number, so that
 # any commit after a tag is considered newer than the semver from the tag
 # without an optional 'v'
-# Note we need `--tags` because github release only creates lightweight tags
+# Note we need `--tags` because github releases only create lightweight tags
 #   (see feature request https://github.com/github/feedback/discussions/4924).
 #   We use `--always` in case of being run as github action with shallow clone.
 #   In this case we might in some situations see an error like
 #   `/bin/bash: line 1: 2b55bbb: value too great for base (error token is "2b55bbb")`
 #   which can be ignored.
-GITDESC := $(shell git describe --tags --always)
+GITDESC := $(shell git describe --tags --always --dirty=-modified 2>/dev/null || true)
+CURRENT_FOLDER_NAME := $(notdir $(CURDIR))
+ifeq ($(strip $(GITDESC)),)
+SEMVER := $(CURRENT_FOLDER_NAME)
+else
 GITDESCPATCH := $(shell echo '$(GITDESC)' | sed -E 's/v?[0-9]+\.[0-9]+\.([0-9]+)[-+]?.*/\1/')
 SEMVERPATCH := $(shell echo $$(( $(GITDESCPATCH) + 1 )))
-# Hint: The regexp in the next line only matches if there is a hyphen (`-`)
+# Hint: The second regexp in the next line only matches
-#       followed by a number, by which we assume that git describe
+#       if there is a hyphen (`-`) followed by a number,
-#       has added a string after the tag
+#       by which we assume that git describe has added a string after the tag
-SEMVER := $(shell echo '$(GITDESC)' | sed -E 's/v?([0-9]+\.[0-9]+\.)([0-9]+)(-[1-9].*)/\1$(SEMVERPATCH)\3/' )
+SEMVER := $(shell echo '$(GITDESC)' | sed -E -e 's/^v//' -e 's/([0-9]+\.[0-9]+\.)([0-9]+)(-[1-9].*)/\1$(SEMVERPATCH)\3/' )
+endif
 testsemver:
 	@echo from \'$(GITDESC)\' transformed to \'$(SEMVER)\'
 
 
 # Set -ldflags parameter to pass the semversion.
-LDFLAGS = -ldflags "-X github.com/csaf-poc/csaf_distribution/v3/util.SemVersion=$(SEMVER)"
+LDFLAGS = -ldflags "-X github.com/gocsaf/csaf/v3/util.SemVersion=$(SEMVER)"
 
 # Build binaries and place them under bin-$(GOOS)-$(GOARCH)
 # Using 'Target-specific Variable Values' to specify the build target system
 
-GOARCH = amd64
+build_linux: GOOS=linux
-build_linux: GOOS = linux
+build_linux: GOARCH=amd64
-build_win: GOOS = windows
-build_mac_amd64: GOOS = darwin
 
-build_mac_arm64: GOARCH = arm64
+build_win: GOOS=windows
-build_mac_arm64: GOOS = darwin
+build_win: GOARCH=amd64
 
-build_linux build_win build_mac_amd64 build_mac_arm64:
+build_mac_amd64: GOOS=darwin
+build_mac_amd64: GOARCH=amd64
+
+build_mac_arm64: GOOS=darwin
+build_mac_arm64: GOARCH=arm64
+
+build_linux_arm64: GOOS=linux
+build_linux_arm64: GOARCH=arm64
+
+build_win_arm64: GOOS=windows
+build_win_arm64: GOARCH=arm64
+
+build_linux build_linux_arm64 build_win build_win_arm64 build_mac_amd64 build_mac_arm64:
 	$(eval BINDIR = bin-$(GOOS)-$(GOARCH)/ )
 	$(MKDIR) $(BINDIR)
 	env GOARCH=$(GOARCH) GOOS=$(GOOS) $(BUILD) -o $(BINDIR) $(LDFLAGS) -v ./cmd/...
 
 
-DISTDIR := csaf_distribution-$(SEMVER)
+DISTDIR := csaf-$(SEMVER)
-dist: build_linux build_win build_mac_amd64 build_mac_arm64
+dist: build_linux build_linux_arm64 build_win build_win_arm64 build_mac_amd64 build_mac_arm64
 	mkdir -p dist
 	mkdir -p dist/$(DISTDIR)-windows-amd64/bin-windows-amd64
+	mkdir -p dist/$(DISTDIR)-windows-arm64/bin-windows-arm64
 	cp README.md dist/$(DISTDIR)-windows-amd64
+	cp README.md dist/$(DISTDIR)-windows-arm64
 	cp bin-windows-amd64/csaf_uploader.exe bin-windows-amd64/csaf_validator.exe \
 	  bin-windows-amd64/csaf_checker.exe bin-windows-amd64/csaf_downloader.exe \
 	  dist/$(DISTDIR)-windows-amd64/bin-windows-amd64/
+	cp bin-windows-arm64/csaf_uploader.exe bin-windows-arm64/csaf_validator.exe \
+	  bin-windows-arm64/csaf_checker.exe bin-windows-arm64/csaf_downloader.exe \
+	  dist/$(DISTDIR)-windows-arm64/bin-windows-arm64/
 	mkdir -p dist/$(DISTDIR)-windows-amd64/docs
+	mkdir -p dist/$(DISTDIR)-windows-arm64/docs
 	cp docs/csaf_uploader.md docs/csaf_validator.md docs/csaf_checker.md \
 	  docs/csaf_downloader.md dist/$(DISTDIR)-windows-amd64/docs
+	cp docs/csaf_uploader.md docs/csaf_validator.md docs/csaf_checker.md \
+	  docs/csaf_downloader.md dist/$(DISTDIR)-windows-arm64/docs
 	mkdir -p dist/$(DISTDIR)-macos/bin-darwin-amd64 \
 		     dist/$(DISTDIR)-macos/bin-darwin-arm64 \
 			 dist/$(DISTDIR)-macos/docs
@@ -98,9 +121,20 @@ dist: build_linux build_win build_mac_amd64 build_mac_arm64
 		cp docs/$${f}.md dist/$(DISTDIR)-macos/docs ; \
 	done
 	mkdir dist/$(DISTDIR)-gnulinux-amd64
-	cp -r README.md docs bin-linux-amd64 dist/$(DISTDIR)-gnulinux-amd64
+	mkdir dist/$(DISTDIR)-gnulinux-arm64
+	cp -r README.md bin-linux-amd64 dist/$(DISTDIR)-gnulinux-amd64
+	cp -r README.md bin-linux-arm64 dist/$(DISTDIR)-gnulinux-arm64
+	# adjust which docs to copy
+	mkdir -p dist/tmp_docs
+	cp -r docs/examples dist/tmp_docs
+	cp docs/*.md dist/tmp_docs
+	cp -r dist/tmp_docs dist/$(DISTDIR)-gnulinux-amd64/docs
+	cp -r dist/tmp_docs dist/$(DISTDIR)-gnulinux-arm64/docs
+	rm -rf dist/tmp_docs
 	cd dist/ ; zip -r $(DISTDIR)-windows-amd64.zip $(DISTDIR)-windows-amd64/
+	cd dist/ ; zip -r $(DISTDIR)-windows-arm64.zip $(DISTDIR)-windows-arm64/
 	cd dist/ ; tar -cvmlzf $(DISTDIR)-gnulinux-amd64.tar.gz $(DISTDIR)-gnulinux-amd64/
+	cd dist/ ; tar -cvmlzf $(DISTDIR)-gnulinux-arm64.tar.gz $(DISTDIR)-gnulinux-arm64/
 	cd dist/ ; tar -cvmlzf $(DISTDIR)-macos.tar.gz $(DISTDIR)-macos
 
 # Remove bin-*-* and dist directories

README.md

@@ -1,7 +1,20 @@
-# csaf_distribution
+<!--
+ This file is Free Software under the Apache-2.0 License
+ without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 
-An implementation of a
+ SPDX-License-Identifier: Apache-2.0
-[CSAF 2.0](https://docs.oasis-open.org/csaf/csaf/v2.0/csaf-v2.0.html)
+
+ SPDX-FileCopyrightText: 2024 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+ Software-Engineering: 2024 Intevation GmbH <https://intevation.de>
+-->
+
+
+
+# csaf
+
+Implements a [CSAF](https://oasis-open.github.io/csaf-documentation/)
+([specification v2.0](https://docs.oasis-open.org/csaf/csaf/v2.0/os/csaf-v2.0-os.html)
+and its [errata](https://docs.oasis-open.org/csaf/csaf/v2.0/csaf-v2.0.html))
 trusted provider, checker, aggregator and downloader.
 Includes an uploader command line tool for the trusted provider.
 
@@ -28,13 +41,20 @@ is a tool for testing a CSAF Trusted Provider according to [Section 7 of the CSA
 ### [csaf_aggregator](docs/csaf_aggregator.md)
 is a CSAF Aggregator, to list or mirror providers.
 
-## Other stuff
+
+## Use as go library
+
+The modules of this repository can be used as library by other Go applications. [ISDuBA](https://github.com/ISDuBA/ISDuBA) does so, for example.
+But there is only limited support and thus it is _not officially supported_.
+There are plans to change this without a concrete schedule within a future major release, e.g. see [#367](https://github.com/gocsaf/csaf/issues/367).
+
+Initially envisioned as a toolbox, it was not constructed as a library,
+and to name one issue, exposes too many functions.
+This leads to problems like [#634](https://github.com/gocsaf/csaf/issues/634), where we have to accept that with 3.2.0 there was an unintended API change.
 
 ### [examples](./examples/README.md)
-are small examples of how to use `github.com/csaf-poc/csaf_distribution`
+are small examples of how to use `github.com/gocsaf/csaf` as an API. Currently this is a work in progress.
-as an API. Currently this is a work in progress, as usage of this repository
+
-as a library to access is _not officially supported_, e.g.
-see https://github.com/csaf-poc/csaf_distribution/issues/367 .
 
 ## Setup
 Binaries for the server side are only available and tested
@@ -58,9 +78,10 @@ Download the binaries from the most recent release assets on Github.
 
 ### Build from sources
 
-- A recent version of **Go** (1.21+) should be installed. [Go installation](https://go.dev/doc/install)
+- Needs a [supported version](docs/Development.md) of **Go** to be installed.
+  [Go installation](https://go.dev/doc/install)
 
-- Clone the repository `git clone https://github.com/csaf-poc/csaf_distribution.git `
+- Clone the repository `git clone https://github.com/gocsaf/csaf.git `
 
 - Build Go components Makefile supplies the following targets:
 	- Build for GNU/Linux system: `make build_linux`
@@ -86,10 +107,18 @@ Binaries will be placed in directories named like `bin-linux-amd64/` and `bin-wi
 
 For further details of the development process consult our [development page](./docs/Development.md).
 
+## Previous repo URLs
+
+> [!NOTE]
+> To avoid future breakage, if you have `csaf-poc` in some of your URLs:
+> 1. Adjust your HTML links.
+> 2. Adjust your go module paths, see [#579](https://github.com/gocsaf/csaf/issues/579#issuecomment-2497244379).
+>
+> (This repository was moved here from https://github.com/csaf-poc/csaf_distribution on 2024-10-28. The old one is deprecated and redirection will be switched off sometime in 2025.)
 
 ## License
 
-- `csaf_distribution` is licensed as Free Software under MIT License.
+- `csaf` is licensed as Free Software under the terms of the [Apache License, Version 2.0](./LICENSES/Apache-2.0.txt).
 
 - See the specific source files
   for details, the license itself can be found in the directory `LICENSES/`.

cmd/csaf_aggregator/client.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -10,23 +10,25 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 var errNotFound = errors.New("not found")
 
 func downloadJSON(c util.Client, url string, found func(io.Reader) error) error {
 	res, err := c.Get(url)
-	if err != nil || res.StatusCode != http.StatusOK ||
+	if err != nil {
+		return fmt.Errorf("not found: %w", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK ||
 		res.Header.Get("Content-Type") != "application/json" {
 		// ignore this as it is expected.
 		return errNotFound
 	}
-	return func() error {
+	return found(res.Body)
-		defer res.Body.Close()
-		return found(res.Body)
-	}()
 }

cmd/csaf_aggregator/client_test.go

@@ -0,0 +1,67 @@
+// This file is Free Software under the MIT License
+// without warranty, see README.md and LICENSES/MIT.txt for details.
+//
+// SPDX-License-Identifier: MIT
+//
+// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
+
+package main
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gocsaf/csaf/v3/util"
+)
+
+func Test_downloadJSON(t *testing.T) {
+	tests := []struct {
+		name        string
+		statusCode  int
+		contentType string
+		wantErr     error
+	}{
+		{
+			name:        "status ok, application/json",
+			statusCode:  http.StatusOK,
+			contentType: "application/json",
+			wantErr:     nil,
+		},
+		{
+			name:        "status found, application/json",
+			statusCode:  http.StatusFound,
+			contentType: "application/json",
+			wantErr:     errNotFound,
+		},
+		{
+			name:        "status ok, application/xml",
+			statusCode:  http.StatusOK,
+			contentType: "application/xml",
+			wantErr:     errNotFound,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			found := func(_ io.Reader) error {
+				return nil
+			}
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.Header().Add("Content-Type", test.contentType)
+				w.WriteHeader(test.statusCode)
+			}))
+			defer server.Close()
+			hClient := http.Client{}
+			client := util.Client(&hClient)
+			if gotErr := downloadJSON(client, server.URL, found); gotErr != test.wantErr {
+				t.Errorf("downloadJSON: Expected %q but got %q.", test.wantErr, gotErr)
+			}
+		})
+	}
+}

cmd/csaf_aggregator/config.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -12,7 +12,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"log"
+	"log/slog"
 	"net/http"
 	"os"
 	"runtime"
@@ -20,12 +20,12 @@ import (
 	"time"
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
+	"github.com/gocsaf/csaf/v3/internal/certs"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/filter"
+	"github.com/gocsaf/csaf/v3/internal/filter"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/models"
+	"github.com/gocsaf/csaf/v3/internal/models"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 	"golang.org/x/time/rate"
 )
 
@@ -178,9 +178,11 @@ func (p *provider) ageAccept(c *config) func(time.Time) bool {
 	}
 
 	if c.Verbose {
-		log.Printf(
+		slog.Debug(
-			"Setting up filter to accept advisories within time range %s to %s\n",
+			"Setting up filter to accept advisories within time range",
-			r[0].Format(time.RFC3339), r[1].Format(time.RFC3339))
+			"from", r[0].Format(time.RFC3339),
+			"to", r[1].Format(time.RFC3339),
+		)
 	}
 	return r.Contains
 }
@@ -262,8 +264,14 @@ func (c *config) privateOpenPGPKey() (*crypto.Key, error) {
 	return c.key, c.keyErr
 }
 
-func (c *config) httpClient(p *provider) util.Client {
+// httpLog does structured logging in a [util.LoggingClient].
+func httpLog(method, url string) {
+	slog.Debug("http",
+		"method", method,
+		"url", url)
+}
 
+func (c *config) httpClient(p *provider) util.Client {
 	hClient := http.Client{}
 
 	var tlsConfig tls.Config
@@ -282,6 +290,7 @@ func (c *config) httpClient(p *provider) util.Client {
 
 	hClient.Transport = &http.Transport{
 		TLSClientConfig: &tlsConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
 
 	client := util.Client(&hClient)
@@ -299,10 +308,18 @@ func (c *config) httpClient(p *provider) util.Client {
 			Client: client,
 			Header: c.ExtraHeader,
 		}
+	default:
+		client = &util.HeaderClient{
+			Client: client,
+			Header: http.Header{},
+		}
 	}
 
 	if c.Verbose {
-		client = &util.LoggingClient{Client: client}
+		client = &util.LoggingClient{
+			Client: client,
+			Log:    httpLog,
+		}
 	}
 
 	if p.Rate == nil && c.Rate == nil {
@@ -323,7 +340,6 @@ func (c *config) httpClient(p *provider) util.Client {
 }
 
 func (c *config) checkProviders() error {
-
 	if !c.AllowSingleProvider && len(c.Providers) < 2 {
 		return errors.New("need at least two providers")
 	}
@@ -393,6 +409,17 @@ func (c *config) setDefaults() {
 	}
 }
 
+// prepareLogging sets up the structured logging.
+func (c *config) prepareLogging() error {
+	ho := slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	}
+	handler := slog.NewTextHandler(os.Stdout, &ho)
+	logger := slog.New(handler)
+	slog.SetDefault(logger)
+	return nil
+}
+
 // compileIgnorePatterns compiles the configured patterns to be ignored.
 func (p *provider) compileIgnorePatterns() error {
 	pm, err := filter.NewPatternMatcher(p.IgnorePattern)
@@ -452,7 +479,6 @@ func (c *config) prepareCertificates() error {
 
 // prepare prepares internal state of a loaded configuration.
 func (c *config) prepare() error {
-
 	if len(c.Providers) == 0 {
 		return errors.New("no providers given in configuration")
 	}

cmd/csaf_aggregator/files.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>

cmd/csaf_aggregator/full.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -11,15 +11,15 @@ package main
 import (
 	"errors"
 	"fmt"
-	"log"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 type fullJob struct {
@@ -29,11 +29,13 @@ type fullJob struct {
 	err                error
 }
 
-// setupProviderFull fetches the provider-metadate.json for a specific provider.
+// setupProviderFull fetches the provider-metadata.json for a specific provider.
 func (w *worker) setupProviderFull(provider *provider) error {
-	log.Printf("worker #%d: %s (%s)\n",
+	w.log.Info("Setting up provider",
-		w.num, provider.Name, provider.Domain)
+		"provider", slog.GroupValue(
-
+			slog.String("name", provider.Name),
+			slog.String("domain", provider.Domain),
+		))
 	w.dir = ""
 	w.provider = provider
 
@@ -55,7 +57,7 @@ func (w *worker) setupProviderFull(provider *provider) error {
 			"provider-metadata.json has %d validation issues", len(errors))
 	}
 
-	log.Printf("provider-metadata: %s\n", w.loc)
+	w.log.Info("Using provider-metadata", "url", w.loc)
 	return nil
 }
 
@@ -79,7 +81,7 @@ func (w *worker) fullWork(wg *sync.WaitGroup, jobs <-chan *fullJob) {
 func (p *processor) full() error {
 
 	if p.cfg.runAsMirror() {
-		log.Println("Running in aggregator mode")
+		p.log.Info("Running in aggregator mode")
 
 		// check if we need to setup a remote validator
 		if p.cfg.RemoteValidatorOptions != nil {
@@ -96,16 +98,18 @@ func (p *processor) full() error {
 			}()
 		}
 	} else {
-		log.Println("Running in lister mode")
+		p.log.Info("Running in lister mode")
 	}
 
 	queue := make(chan *fullJob)
 	var wg sync.WaitGroup
 
-	log.Printf("Starting %d workers.\n", p.cfg.Workers)
+	p.log.Info("Starting workers...", "num", p.cfg.Workers)
+
 	for i := 1; i <= p.cfg.Workers; i++ {
 		wg.Add(1)
 		w := newWorker(i, p)
+
 		go w.fullWork(&wg, queue)
 	}
 
@@ -135,12 +139,22 @@ func (p *processor) full() error {
 	for i := range jobs {
 		j := &jobs[i]
 		if j.err != nil {
-			log.Printf("error: '%s' failed: %v\n", j.provider.Name, j.err)
+			p.log.Error("Job execution failed",
+				slog.Group("job",
+					slog.Group("provider"),
+					"name", j.provider.Name,
+				),
+				"err", j.err,
+			)
 			continue
 		}
 		if j.aggregatorProvider == nil {
-			log.Printf(
+			p.log.Error("Job did not produce any result",
-				"error: '%s' does not produce any result.\n", j.provider.Name)
+				slog.Group("job",
+					slog.Group("provider"),
+					"name", j.provider.Name,
+				),
+			)
 			continue
 		}
 

cmd/csaf_aggregator/indices.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -12,7 +12,6 @@ import (
 	"bufio"
 	"encoding/csv"
 	"fmt"
-	"log"
 	"os"
 	"path/filepath"
 	"sort"
@@ -20,8 +19,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 const (
@@ -184,19 +183,26 @@ func (w *worker) writeROLIENoSummaries(label string) error {
 
 	fname := "csaf-feed-tlp-" + labelFolder + ".json"
 
-	feedURL := w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" +
+	feedURL, err := w.getProviderBaseURL()
-		w.provider.Name + "/" + labelFolder + "/" + fname
+	if err != nil {
+		return err
+	}
+	feedURL = feedURL.JoinPath(labelFolder, fname)
 
 	links := []csaf.Link{{
 		Rel:  "self",
-		HRef: feedURL,
+		HRef: feedURL.String(),
 	}}
 
 	if w.provider.serviceDocument(w.processor.cfg) {
+		serviceURL, err := w.getProviderBaseURL()
+		if err != nil {
+			return err
+		}
+		serviceURL = serviceURL.JoinPath("service.json")
 		links = append(links, csaf.Link{
-			Rel: "service",
+			Rel:  "service",
-			HRef: w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" +
+			HRef: serviceURL.String(),
-				w.provider.Name + "/service.json",
 		})
 	}
 
@@ -224,8 +230,11 @@ func (w *worker) writeROLIE(label string, summaries []summary) error {
 
 	fname := "csaf-feed-tlp-" + labelFolder + ".json"
 
-	feedURL := w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" +
+	feedURL, err := w.getProviderBaseURL()
-		w.provider.Name + "/" + labelFolder + "/" + fname
+	if err != nil {
+		return err
+	}
+	feedURL = feedURL.JoinPath(labelFolder, fname)
 
 	entries := make([]*csaf.Entry, len(summaries))
 
@@ -237,10 +246,13 @@ func (w *worker) writeROLIE(label string, summaries []summary) error {
 	for i := range summaries {
 		s := &summaries[i]
 
-		csafURL := w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" +
+		csafURL, err := w.getProviderBaseURL()
-			w.provider.Name + "/" + label + "/" +
+		if err != nil {
-			strconv.Itoa(s.summary.InitialReleaseDate.Year()) + "/" +
+			return err
-			s.filename
+		}
+		csafURLString := csafURL.JoinPath(label,
+			strconv.Itoa(s.summary.InitialReleaseDate.Year()),
+			s.filename).String()
 
 		entries[i] = &csaf.Entry{
 			ID:        s.summary.ID,
@@ -248,15 +260,15 @@ func (w *worker) writeROLIE(label string, summaries []summary) error {
 			Published: csaf.TimeStamp(s.summary.InitialReleaseDate),
 			Updated:   csaf.TimeStamp(s.summary.CurrentReleaseDate),
 			Link: []csaf.Link{
-				{Rel: "self", HRef: csafURL},
+				{Rel: "self", HRef: csafURLString},
-				{Rel: "hash", HRef: csafURL + ".sha256"},
+				{Rel: "hash", HRef: csafURLString + ".sha256"},
-				{Rel: "hash", HRef: csafURL + ".sha512"},
+				{Rel: "hash", HRef: csafURLString + ".sha512"},
-				{Rel: "signature", HRef: csafURL + ".asc"},
+				{Rel: "signature", HRef: csafURLString + ".asc"},
 			},
 			Format: format,
 			Content: csaf.Content{
 				Type: "application/json",
-				Src:  csafURL,
+				Src:  csafURLString,
 			},
 		}
 		if s.summary.Summary != "" {
@@ -268,14 +280,18 @@ func (w *worker) writeROLIE(label string, summaries []summary) error {
 
 	links := []csaf.Link{{
 		Rel:  "self",
-		HRef: feedURL,
+		HRef: feedURL.String(),
 	}}
 
 	if w.provider.serviceDocument(w.processor.cfg) {
+		serviceURL, err := w.getProviderBaseURL()
+		if err != nil {
+			return err
+		}
+		serviceURL = serviceURL.JoinPath("service.json")
 		links = append(links, csaf.Link{
-			Rel: "service",
+			Rel:  "service",
-			HRef: w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" +
+			HRef: serviceURL.String(),
-				w.provider.Name + "/service.json",
 		})
 	}
 
@@ -345,12 +361,15 @@ func (w *worker) writeService() error {
 	for _, ts := range labels {
 		feedName := "csaf-feed-tlp-" + ts + ".json"
 
-		href := w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" +
+		hrefURL, err := w.getProviderBaseURL()
-			w.provider.Name + "/" + ts + "/" + feedName
+		if err != nil {
+			return err
+		}
+		hrefURL = hrefURL.JoinPath(ts, feedName)
 
 		collection := csaf.ROLIEServiceWorkspaceCollection{
 			Title:      "CSAF feed (TLP:" + strings.ToUpper(ts) + ")",
-			HRef:       href,
+			HRef:       hrefURL.String(),
 			Categories: categories,
 		}
 		collections = append(collections, collection)
@@ -377,7 +396,7 @@ func (w *worker) writeIndices() error {
 	}
 
 	for label, summaries := range w.summaries {
-		log.Printf("%s: %d\n", label, len(summaries))
+		w.log.Debug("Writing indices", "label", label, "summaries.num", len(summaries))
 		if err := w.writeInterims(label, summaries); err != nil {
 			return err
 		}

cmd/csaf_aggregator/interim.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -13,11 +13,9 @@ import (
 	"crypto/sha256"
 	"crypto/sha512"
 	"encoding/csv"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -25,8 +23,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 type interimJob struct {
@@ -82,7 +81,7 @@ func (w *worker) checkInterims(
 		if err := func() error {
 			defer res.Body.Close()
 			tee := io.TeeReader(res.Body, hasher)
-			return json.NewDecoder(tee).Decode(&doc)
+			return misc.StrictJSONParse(tee, &doc)
 		}(); err != nil {
 			return nil, err
 		}
@@ -102,12 +101,12 @@ func (w *worker) checkInterims(
 
 		// XXX: Should we return an error here?
 		for _, e := range errors {
-			log.Printf("validation error: %s: %v\n", url, e)
+			w.log.Error("validation error", "url", url, "err", e)
 		}
 
 		// We need to write the changed content.
 
-		// This will start the transcation if not already started.
+		// This will start the transaction if not already started.
 		dst, err := tx.Dst()
 		if err != nil {
 			return nil, err
@@ -159,8 +158,7 @@ func (w *worker) checkInterims(
 
 // setupProviderInterim prepares the worker for a specific provider.
 func (w *worker) setupProviderInterim(provider *provider) {
-	log.Printf("worker #%d: %s (%s)\n",
+	w.log.Info("Setting up worker", provider.Name, provider.Domain)
-		w.num, provider.Name, provider.Domain)
 
 	w.dir = ""
 	w.provider = provider
@@ -262,7 +260,7 @@ func (p *processor) interim() error {
 	queue := make(chan *interimJob)
 	var wg sync.WaitGroup
 
-	log.Printf("Starting %d workers.\n", p.cfg.Workers)
+	p.log.Info("Starting workers...", "num", p.cfg.Workers)
 	for i := 1; i <= p.cfg.Workers; i++ {
 		wg.Add(1)
 		w := newWorker(i, p)

cmd/csaf_aggregator/lazytransaction.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -9,11 +9,11 @@
 package main
 
 import (
-	"log"
+	"log/slog"
 	"os"
 	"path/filepath"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 type lazyTransaction struct {
@@ -85,7 +85,8 @@ func (lt *lazyTransaction) commit() error {
 		os.RemoveAll(lt.dst)
 		return err
 	}
-	log.Printf("Move %q -> %q\n", symlink, lt.src)
+
+	slog.Debug("Moving directory", "from", symlink, "to", lt.src)
 	if err := os.Rename(symlink, lt.src); err != nil {
 		os.RemoveAll(lt.dst)
 		return err

cmd/csaf_aggregator/lister.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -11,8 +11,8 @@ package main
 import (
 	"fmt"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // mirrorAllowed checks if mirroring is allowed.

cmd/csaf_aggregator/main.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -11,10 +11,12 @@ package main
 
 import (
 	"fmt"
+	"log/slog"
 	"os"
 	"path/filepath"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
+
 	"github.com/gofrs/flock"
 )
 
@@ -44,8 +46,9 @@ func lock(lockFile *string, fn func() error) error {
 
 func main() {
 	_, cfg, err := parseArgsConfig()
-	options.ErrorCheck(err)
+	cfg.prepareLogging()
-	options.ErrorCheck(cfg.prepare())
+	options.ErrorCheckStructured(err)
-	p := processor{cfg: cfg}
+	options.ErrorCheckStructured(cfg.prepare())
-	options.ErrorCheck(lock(cfg.LockFile, p.process))
+	p := processor{cfg: cfg, log: slog.Default()}
+	options.ErrorCheckStructured(lock(cfg.LockFile, p.process))
 }

cmd/csaf_aggregator/mirror.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -13,10 +13,9 @@ import (
 	"crypto/sha256"
 	"crypto/sha512"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
 	"io"
-	"log"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"os"
@@ -30,8 +29,9 @@ import (
 	"github.com/ProtonMail/gopenpgp/v2/constants"
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // mirrorAllowed checks if mirroring is allowed.
@@ -47,7 +47,7 @@ func (w *worker) mirror() (*csaf.AggregatorCSAFProvider, error) {
 	if err != nil && w.dir != "" {
 		// If something goes wrong remove the debris.
 		if err := os.RemoveAll(w.dir); err != nil {
-			log.Printf("error: %v\n", err)
+			w.log.Error("Could not remove directory", "path", w.dir, "err", err)
 		}
 	}
 	return result, err
@@ -67,7 +67,7 @@ func (w *worker) mirrorInternal() (*csaf.AggregatorCSAFProvider, error) {
 	// Collecting the categories per label.
 	w.categories = map[string]util.Set[string]{}
 
-	base, err := url.Parse(w.loc)
+	pmdURL, err := url.Parse(w.loc)
 	if err != nil {
 		return nil, err
 	}
@@ -76,7 +76,7 @@ func (w *worker) mirrorInternal() (*csaf.AggregatorCSAFProvider, error) {
 		w.client,
 		w.expr,
 		w.metadataProvider,
-		base)
+		pmdURL)
 
 	afp.AgeAccept = w.provider.ageAccept(w.processor.cfg)
 
@@ -103,9 +103,13 @@ func (w *worker) mirrorInternal() (*csaf.AggregatorCSAFProvider, error) {
 	}
 
 	// Add us as a mirror.
+	mirror, err := w.getProviderBaseURL()
+	if err != nil {
+		return nil, err
+	}
 	mirrorURL := csaf.ProviderURL(
-		fmt.Sprintf("%s/.well-known/csaf-aggregator/%s/provider-metadata.json",
+		mirror.JoinPath("provider-metadata.json").String(),
-			w.processor.cfg.Domain, w.provider.Name))
+	)
 
 	acp.Mirrors = []csaf.ProviderURL{
 		mirrorURL,
@@ -128,8 +132,12 @@ func (w *worker) writeProviderMetadata() error {
 
 	fname := filepath.Join(w.dir, "provider-metadata.json")
 
+	prefixURL, err := w.getProviderBaseURL()
+	if err != nil {
+		return err
+	}
 	pm := csaf.NewProviderMetadataPrefix(
-		w.processor.cfg.Domain+"/.well-known/csaf-aggregator/"+w.provider.Name,
+		prefixURL.String(),
 		w.labelsFromSummaries())
 
 	// Fill in directory URLs if needed.
@@ -139,9 +147,8 @@ func (w *worker) writeProviderMetadata() error {
 			labels = append(labels, label)
 		}
 		sort.Strings(labels)
-		prefix := w.processor.cfg.Domain + "/.well-known/csaf-aggregator/" + w.provider.Name + "/"
 		for _, label := range labels {
-			pm.AddDirectoryDistribution(prefix + label)
+			pm.AddDirectoryDistribution(prefixURL.JoinPath(label).String())
 		}
 	}
 
@@ -166,7 +173,7 @@ func (w *worker) writeProviderMetadata() error {
 		{Expr: `$.public_openpgp_keys`, Action: util.ReMarshalMatcher(&pm.PGPKeys)},
 	}, w.metadataProvider); err != nil {
 		// only log the errors
-		log.Printf("extracting data from orignal provider failed: %v\n", err)
+		w.log.Error("Extracting data from original provider failed", "err", err)
 	}
 
 	// We are mirroring the remote public keys, too.
@@ -188,19 +195,22 @@ func (w *worker) mirrorPGPKeys(pm *csaf.ProviderMetadata) error {
 		return err
 	}
 
+	keyURL, err := w.getProviderBaseURL()
+	if err != nil {
+		return err
+	}
 	localKeyURL := func(fingerprint string) string {
-		return fmt.Sprintf("%s/.well-known/csaf-aggregator/%s/openpgp/%s.asc",
+		return keyURL.JoinPath("openpgp", (fingerprint + ".asc")).String()
-			w.processor.cfg.Domain, w.provider.Name, fingerprint)
 	}
 
 	for i := range pm.PGPKeys {
 		pgpKey := &pm.PGPKeys[i]
 		if pgpKey.URL == nil {
-			log.Printf("ignoring PGP key without URL: %s\n", pgpKey.Fingerprint)
+			w.log.Warn("Ignoring PGP key without URL", "fingerprint", pgpKey.Fingerprint)
 			continue
 		}
 		if _, err := hex.DecodeString(string(pgpKey.Fingerprint)); err != nil {
-			log.Printf("ignoring PGP with invalid fingerprint: %s\n", *pgpKey.URL)
+			w.log.Warn("Ignoring PGP key with invalid fingerprint", "url", *pgpKey.URL)
 			continue
 		}
 
@@ -240,8 +250,8 @@ func (w *worker) mirrorPGPKeys(pm *csaf.ProviderMetadata) error {
 		}
 
 		// replace the URL
-		url := localKeyURL(fingerprint)
+		u := localKeyURL(fingerprint)
-		pgpKey.URL = &url
+		pgpKey.URL = &u
 	}
 
 	// If we have public key configured copy it into the new folder
@@ -308,7 +318,7 @@ func (w *worker) createAggregatorProvider() (*csaf.AggregatorCSAFProvider, error
 	var (
 		lastUpdated = csaf.TimeStamp(lastUpdatedT)
 		role        = csaf.MetadataRole(roleS)
-		url         = csaf.ProviderURL(urlS)
+		providerURL = csaf.ProviderURL(urlS)
 	)
 
 	return &csaf.AggregatorCSAFProvider{
@@ -316,7 +326,7 @@ func (w *worker) createAggregatorProvider() (*csaf.AggregatorCSAFProvider, error
 			LastUpdated: &lastUpdated,
 			Publisher:   &pub,
 			Role:        &role,
-			URL:         &url,
+			URL:         &providerURL,
 		},
 	}, nil
 }
@@ -344,7 +354,7 @@ func (w *worker) doMirrorTransaction() error {
 
 	// Check if there is a sysmlink already.
 	target := filepath.Join(w.processor.cfg.Folder, w.provider.Name)
-	log.Printf("target: '%s'\n", target)
+	w.log.Debug("Checking for path existance", "path", target)
 
 	exists, err := util.PathExists(target)
 	if err != nil {
@@ -359,7 +369,7 @@ func (w *worker) doMirrorTransaction() error {
 		}
 	}
 
-	log.Printf("sym link: %s -> %s\n", w.dir, target)
+	w.log.Debug("Creating sym link", "from", w.dir, "to", target)
 
 	// Create a new symlink
 	if err := os.Symlink(w.dir, target); err != nil {
@@ -368,7 +378,7 @@ func (w *worker) doMirrorTransaction() error {
 	}
 
 	// Move the symlink
-	log.Printf("Move: %s -> %s\n", target, webTarget)
+	w.log.Debug("Moving sym link", "from", target, "to", webTarget)
 	if err := os.Rename(target, webTarget); err != nil {
 		os.RemoveAll(w.dir)
 		return err
@@ -462,8 +472,9 @@ func (w *worker) extractCategories(label string, advisory any) error {
 			expr := cat[len(exprPrefix):]
 			// Compile first to check that the expression is okay.
 			if _, err := w.expr.Compile(expr); err != nil {
-				fmt.Printf("Compiling category expression %q failed: %v\n",
+				slog.Error("Compiling category expression failed",
-					expr, err)
+					"expr", expr,
+					"err", err)
 				continue
 			}
 			// Ignore errors here as they result from not matching.
@@ -499,14 +510,14 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 
 		u, err := url.Parse(file.URL())
 		if err != nil {
-			log.Printf("error: %s\n", err)
+			w.log.Error("Could not parse advisory file URL", "err", err)
 			continue
 		}
 
 		// Should we ignore this advisory?
 		if w.provider.ignoreURL(file.URL(), w.processor.cfg) {
 			if w.processor.cfg.Verbose {
-				log.Printf("Ignoring %s: %q\n", w.provider.Name, file.URL())
+				w.log.Info("Ignoring advisory", slog.Group("provider", "name", w.provider.Name), "file", file)
 			}
 			continue
 		}
@@ -514,7 +525,7 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 		// Ignore not conforming filenames.
 		filename := filepath.Base(u.Path)
 		if !util.ConformingFileName(filename) {
-			log.Printf("Not conforming filename %q. Ignoring.\n", filename)
+			w.log.Warn("Ignoring advisory because of non-conforming filename", "filename", filename)
 			continue
 		}
 
@@ -527,23 +538,22 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 
 		download := func(r io.Reader) error {
 			tee := io.TeeReader(r, hasher)
-			return json.NewDecoder(tee).Decode(&advisory)
+			return misc.StrictJSONParse(tee, &advisory)
 		}
 
 		if err := downloadJSON(w.client, file.URL(), download); err != nil {
-			log.Printf("error: %v\n", err)
+			w.log.Error("Error while downloading JSON", "err", err)
 			continue
 		}
 
 		// Check against CSAF schema.
 		errors, err := csaf.ValidateCSAF(advisory)
 		if err != nil {
-			log.Printf("error: %s: %v", file, err)
+			w.log.Error("Error while validating CSAF schema", "err", err)
 			continue
 		}
 		if len(errors) > 0 {
-			log.Printf("CSAF file %s has %d validation errors.\n",
+			w.log.Error("CSAF file has validation errors", "num.errors", len(errors), "file", file)
-				file, len(errors))
 			continue
 		}
 
@@ -551,29 +561,27 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 		if rmv := w.processor.remoteValidator; rmv != nil {
 			rvr, err := rmv.Validate(advisory)
 			if err != nil {
-				log.Printf("Calling remote validator failed: %s\n", err)
+				w.log.Error("Calling remote validator failed", "err", err)
 				continue
 			}
 			if !rvr.Valid {
-				log.Printf(
+				w.log.Error("CSAF file does not validate remotely", "file", file.URL())
-					"CSAF file %s does not validate remotely.\n", file)
 				continue
 			}
 		}
 
 		sum, err := csaf.NewAdvisorySummary(w.expr, advisory)
 		if err != nil {
-			log.Printf("error: %s: %v\n", file, err)
+			w.log.Error("Error while creating new advisory", "file", file, "err", err)
 			continue
 		}
 
 		if util.CleanFileName(sum.ID) != filename {
-			log.Printf("ID %q does not match filename %s",
+			w.log.Error("ID mismatch", "id", sum.ID, "filename", filename)
-				sum.ID, filename)
 		}
 
 		if err := w.extractCategories(label, advisory); err != nil {
-			log.Printf("error: %s: %v\n", file, err)
+			w.log.Error("Could not extract categories", "file", file, "err", err)
 			continue
 		}
 
@@ -591,12 +599,10 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 			if err := os.MkdirAll(yearDir, 0755); err != nil {
 				return err
 			}
-			//log.Printf("created %s\n", yearDir)
 			yearDirs[year] = yearDir
 		}
 
 		fname := filepath.Join(yearDir, filename)
-		//log.Printf("write: %s\n", fname)
 		data := content.Bytes()
 		if err := writeFileHashes(
 			fname, filename,
@@ -621,10 +627,9 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile)
 // If this fails it creates a signature itself with the configured key.
 func (w *worker) downloadSignatureOrSign(url, fname string, data []byte) error {
 	sig, err := w.downloadSignature(url)
-
 	if err != nil {
 		if err != errNotFound {
-			log.Printf("error: %s: %v\n", url, err)
+			w.log.Error("Could not find signature URL", "url", url, "err", err)
 		}
 		// Sign it our self.
 		if sig, err = w.sign(data); err != nil {

cmd/csaf_aggregator/processor.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -10,14 +10,15 @@ package main
 
 import (
 	"fmt"
-	"log"
+	"log/slog"
+	"net/url"
 	"os"
 	"path/filepath"
 
-	"github.com/ProtonMail/gopenpgp/v2/crypto"
+	"github.com/gocsaf/csaf/v3/csaf"
+	"github.com/gocsaf/csaf/v3/util"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/ProtonMail/gopenpgp/v2/crypto"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
 )
 
 type processor struct {
@@ -26,6 +27,9 @@ type processor struct {
 
 	// remoteValidator is a globally configured remote validator.
 	remoteValidator csaf.RemoteValidator
+
+	// log is the structured logger for the whole processor.
+	log *slog.Logger
 }
 
 type summary struct {
@@ -48,6 +52,7 @@ type worker struct {
 	dir              string                      // Directory to store data to.
 	summaries        map[string][]summary        // the summaries of the advisories.
 	categories       map[string]util.Set[string] // the categories per label.
+	log              *slog.Logger                // the structured logger, supplied with the worker number.
 }
 
 func newWorker(num int, processor *processor) *worker {
@@ -55,6 +60,7 @@ func newWorker(num int, processor *processor) *worker {
 		num:       num,
 		processor: processor,
 		expr:      util.NewPathEval(),
+		log:       processor.log.With(slog.Int("worker", num)),
 	}
 }
 
@@ -84,16 +90,21 @@ func (w *worker) locateProviderMetadata(domain string) error {
 
 	lpmd := loader.Load(domain)
 
-	if w.processor.cfg.Verbose {
-		for i := range lpmd.Messages {
-			log.Printf(
-				"Loading provider-metadata.json of %q: %s\n",
-				domain, lpmd.Messages[i].Message)
-		}
-	}
-
 	if !lpmd.Valid() {
+		for i := range lpmd.Messages {
+			w.log.Error(
+				"Loading provider-metadata.json",
+				"domain", domain,
+				"message", lpmd.Messages[i].Message)
+		}
 		return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain)
+	} else if w.processor.cfg.Verbose {
+		for i := range lpmd.Messages {
+			w.log.Debug(
+				"Loading provider-metadata.json",
+				"domain", domain,
+				"message", lpmd.Messages[i].Message)
+		}
 	}
 
 	w.metadataProvider = lpmd.Document
@@ -102,6 +113,18 @@ func (w *worker) locateProviderMetadata(domain string) error {
 	return nil
 }
 
+// getProviderBaseURL returns the base URL for the provider.
+func (w *worker) getProviderBaseURL() (*url.URL, error) {
+	baseURL, err := url.Parse(w.processor.cfg.Domain)
+	if err != nil {
+		return nil, err
+	}
+	baseURL = baseURL.JoinPath(".well-known",
+		"csaf-aggregator",
+		w.provider.Name)
+	return baseURL, nil
+}
+
 // removeOrphans removes the directories that are not in the providers list.
 func (p *processor) removeOrphans() error {
 
@@ -141,7 +164,7 @@ func (p *processor) removeOrphans() error {
 
 		fi, err := entry.Info()
 		if err != nil {
-			log.Printf("error: %v\n", err)
+			p.log.Error("Could not retrieve file info", "err", err)
 			continue
 		}
 
@@ -153,13 +176,13 @@ func (p *processor) removeOrphans() error {
 		d := filepath.Join(path, entry.Name())
 		r, err := filepath.EvalSymlinks(d)
 		if err != nil {
-			log.Printf("error: %v\n", err)
+			p.log.Error("Could not evaluate symlink", "err", err)
 			continue
 		}
 
 		fd, err := os.Stat(r)
 		if err != nil {
-			log.Printf("error: %v\n", err)
+			p.log.Error("Could not retrieve file stats", "err", err)
 			continue
 		}
 
@@ -169,18 +192,18 @@ func (p *processor) removeOrphans() error {
 		}
 
 		// Remove the link.
-		log.Printf("removing link %s -> %s\n", d, r)
+		p.log.Info("Removing link", "path", fmt.Sprintf("%s -> %s", d, r))
 		if err := os.Remove(d); err != nil {
-			log.Printf("error: %v\n", err)
+			p.log.Error("Could not remove symlink", "err", err)
 			continue
 		}
 
 		// Only remove directories which are in our folder.
 		if rel, err := filepath.Rel(prefix, r); err == nil &&
 			rel == filepath.Base(r) {
-			log.Printf("removing directory %s\n", r)
+			p.log.Info("Remove directory", "path", r)
 			if err := os.RemoveAll(r); err != nil {
-				log.Printf("error: %v\n", err)
+				p.log.Error("Could not remove directory", "err", err)
 			}
 		}
 	}

cmd/csaf_checker/config.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -13,10 +13,10 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
+	"github.com/gocsaf/csaf/v3/internal/certs"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/filter"
+	"github.com/gocsaf/csaf/v3/internal/filter"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/models"
+	"github.com/gocsaf/csaf/v3/internal/models"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
 )
 
 type outputFormat string

cmd/csaf_checker/links.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -13,9 +13,11 @@ import (
 	"net/http"
 	"net/url"
 
+	"github.com/gocsaf/csaf/v3/internal/misc"
+
 	"github.com/PuerkitoBio/goquery"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 type (
@@ -93,7 +95,12 @@ func (pgs pages) listed(
 				return err
 			}
 			// Links may be relative
-			abs := baseURL.ResolveReference(u).String()
+			var abs string
+			if u.IsAbs() {
+				abs = u.String()
+			} else {
+				abs = misc.JoinURL(baseURL, u).String()
+			}
 			content.links.Add(abs)
 			return nil
 		})

cmd/csaf_checker/links_test.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -10,8 +10,12 @@ package main
 
 import (
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 const page0 = `<html>
@@ -31,7 +35,6 @@ const page0 = `<html>
 </html>`
 
 func TestLinksOnPage(t *testing.T) {
-
 	var links []string
 
 	err := linksOnPage(
@@ -58,3 +61,78 @@ func TestLinksOnPage(t *testing.T) {
 		}
 	}
 }
+
+func Test_listed(t *testing.T) {
+	tests := []struct {
+		name    string
+		badDirs util.Set[string]
+		path    string
+		want    bool
+	}{
+		{
+			name:    "listed path",
+			badDirs: util.Set[string]{},
+			path:    "/white/avendor-advisory-0004.json",
+			want:    true,
+		},
+		{
+			name:    "badDirs contains path",
+			badDirs: util.Set[string]{"/white/": {}},
+			path:    "/white/avendor-advisory-0004.json",
+			want:    false,
+		},
+		{
+			name:    "not found",
+			badDirs: util.Set[string]{},
+			path:    "/not-found/resource.json",
+			want:    false,
+		},
+		{
+			name:    "badDirs does not contain path",
+			badDirs: util.Set[string]{"/bad-dir/": {}},
+			path:    "/white/avendor-advisory-0004.json",
+			want:    true,
+		},
+		{
+			name:    "unlisted path",
+			badDirs: util.Set[string]{},
+			path:    "/white/avendor-advisory-0004-not-listed.json",
+			want:    false,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			serverURL := ""
+			fs := http.FileServer(http.Dir("../../testdata/simple-directory-provider"))
+			server := httptest.NewTLSServer(fs)
+			defer server.Close()
+
+			serverURL = server.URL
+
+			hClient := server.Client()
+			client := util.Client(hClient)
+
+			pgs := pages{}
+			cfg := config{RemoteValidator: "", RemoteValidatorCache: ""}
+			p, err := newProcessor(&cfg)
+			if err != nil {
+				t.Error(err)
+			}
+			p.client = client
+
+			badDirs := util.Set[string]{}
+			for dir := range test.badDirs {
+				badDirs.Add(serverURL + dir)
+			}
+
+			got, _ := pgs.listed(serverURL+test.path, p, badDirs)
+			if got != test.want {
+				t.Errorf("%q: Expected %t but got %t.", test.name, test.want, got)
+			}
+		})
+	}
+}

cmd/csaf_checker/main.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -12,7 +12,7 @@ package main
 import (
 	"log"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
 )
 
 // run uses a processor to check all the given domains or direct urls

cmd/csaf_checker/processor.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -15,7 +15,6 @@ import (
 	"crypto/sha512"
 	"crypto/tls"
 	"encoding/csv"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -29,12 +28,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gocsaf/csaf/v3/internal/misc"
+
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 	"golang.org/x/time/rate"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/models"
+	"github.com/gocsaf/csaf/v3/util"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
 )
 
 // topicMessages stores the collected topicMessages for a specific topic.
@@ -54,6 +54,8 @@ type processor struct {
 	pmd            any
 	keys           *crypto.KeyRing
 	labelChecker   labelChecker
+	timesChanges   map[string]time.Time
+	timesAdv       map[string]time.Time
 
 	invalidAdvisories      topicMessages
 	badFilenames           topicMessages
@@ -84,10 +86,8 @@ type reporter interface {
 	report(*processor, *Domain)
 }
 
-var (
+// errContinue indicates that the current check should continue.
-	// errContinue indicates that the current check should continue.
+var errContinue = errors.New("continue")
-	errContinue = errors.New("continue")
-)
 
 type whereType byte
 
@@ -139,7 +139,7 @@ func (m *topicMessages) info(format string, args ...any) {
 	m.add(InfoType, format, args...)
 }
 
-// use signals that we going to use this topic.
+// use signals that we're going to use this topic.
 func (m *topicMessages) use() {
 	if *m == nil {
 		*m = []Message{}
@@ -165,9 +165,8 @@ func (m *topicMessages) hasErrors() bool {
 	return false
 }
 
-// newProcessor returns an initilaized processor.
+// newProcessor returns an initialized processor.
 func newProcessor(cfg *config) (*processor, error) {
-
 	var validator csaf.RemoteValidator
 
 	if cfg.RemoteValidator != "" {
@@ -192,6 +191,9 @@ func newProcessor(cfg *config) (*processor, error) {
 			advisories:      map[csaf.TLPLabel]util.Set[string]{},
 			whiteAdvisories: map[identifier]bool{},
 		},
+		timesAdv:     map[string]time.Time{},
+		timesChanges: map[string]time.Time{},
+		noneTLS:      util.Set[string]{},
 	}, nil
 }
 
@@ -203,17 +205,17 @@ func (p *processor) close() {
 	}
 }
 
-// clean clears the fields values of the given processor.
+// reset clears the fields values of the given processor.
-func (p *processor) clean() {
+func (p *processor) reset() {
 	p.redirects = nil
-	p.noneTLS = nil
-	for k := range p.alreadyChecked {
-		delete(p.alreadyChecked, k)
-	}
 	p.pmdURL = ""
 	p.pmd256 = nil
 	p.pmd = nil
 	p.keys = nil
+	clear(p.alreadyChecked)
+	clear(p.noneTLS)
+	clear(p.timesAdv)
+	clear(p.timesChanges)
 
 	p.invalidAdvisories.reset()
 	p.badFilenames.reset()
@@ -240,7 +242,6 @@ func (p *processor) clean() {
 // Then it calls the report method on each report from the given "reporters" parameter for each domain.
 // It returns a pointer to the report and nil, otherwise an error.
 func (p *processor) run(domains []string) (*Report, error) {
-
 	report := Report{
 		Date:      ReportTime{Time: time.Now().UTC()},
 		Version:   util.SemVersion,
@@ -248,15 +249,17 @@ func (p *processor) run(domains []string) (*Report, error) {
 	}
 
 	for _, d := range domains {
+		p.reset()
+
 		if !p.checkProviderMetadata(d) {
-			// We cannot build a report if the provider metadata cannot be parsed.
+			// We need to fail the domain if the PMD cannot be parsed.
-			log.Printf("Could not parse the Provider-Metadata.json of: %s\n", d)
+			p.badProviderMetadata.use()
-			continue
+			p.badProviderMetadata.error("Could not parse the Provider-Metadata.json of: %s", d)
+
 		}
 		if err := p.checkDomain(d); err != nil {
-			log.Printf("Failed to find valid provider-metadata.json for domain %s: %v. "+
+			p.badProviderMetadata.use()
-				"Continuing with next domain.", d, err)
+			p.badProviderMetadata.error("Failed to find valid provider-metadata.json for domain %s: %v. ", d, err)
-			continue
 		}
 		domain := &Domain{Name: d}
 
@@ -267,8 +270,10 @@ func (p *processor) run(domains []string) (*Report, error) {
 		}
 
 		if domain.Role == nil {
-			log.Printf("No role found in meta data. Ignoring domain %q\n", d)
+			log.Printf("No role found in meta data for domain %q\n", d)
-			continue
+			// Assume trusted provider to continue report generation
+			role := csaf.MetadataRoleTrustedProvider
+			domain.Role = &role
 		}
 
 		rules := roleRequirements(*domain.Role)
@@ -288,7 +293,6 @@ func (p *processor) run(domains []string) (*Report, error) {
 		domain.Passed = rules.eval(p)
 
 		report.Domains = append(report.Domains, domain)
-		p.clean()
 	}
 
 	return &report, nil
@@ -296,7 +300,6 @@ func (p *processor) run(domains []string) (*Report, error) {
 
 // fillMeta fills the report with extra informations from provider metadata.
 func (p *processor) fillMeta(domain *Domain) error {
-
 	if p.pmd == nil {
 		return nil
 	}
@@ -322,7 +325,6 @@ func (p *processor) fillMeta(domain *Domain) error {
 // domainChecks compiles a list of checks which should be performed
 // for a given domain.
 func (p *processor) domainChecks(domain string) []func(*processor, string) error {
-
 	// If we have a direct domain url we dont need to
 	// perform certain checks.
 	direct := strings.HasPrefix(domain, "https://")
@@ -377,9 +379,6 @@ func (p *processor) checkDomain(domain string) error {
 // checkTLS parses the given URL to check its schema, as a result it sets
 // the value of "noneTLS" field if it is not HTTPS.
 func (p *processor) checkTLS(u string) {
-	if p.noneTLS == nil {
-		p.noneTLS = util.Set[string]{}
-	}
 	if x, err := url.Parse(u); err == nil && x.Scheme != "https" {
 		p.noneTLS.Add(u)
 	}
@@ -392,7 +391,6 @@ func (p *processor) markChecked(s string, mask whereType) bool {
 }
 
 func (p *processor) checkRedirect(r *http.Request, via []*http.Request) error {
-
 	url := r.URL.String()
 	p.checkTLS(url)
 	if p.redirects == nil {
@@ -430,16 +428,15 @@ func (p *processor) fullClient() util.Client {
 
 	hClient.Transport = &http.Transport{
 		TLSClientConfig: &tlsConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
 
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(p.cfg.ExtraHeader) > 0 {
+	client = &util.HeaderClient{
-		client = &util.HeaderClient{
+		Client: client,
-			Client: client,
+		Header: p.cfg.ExtraHeader,
-			Header: p.cfg.ExtraHeader,
-		}
 	}
 
 	// Add optional URL logging.
@@ -462,6 +459,7 @@ func (p *processor) basicClient() *http.Client {
 	if p.cfg.Insecure {
 		tr := &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			Proxy:           http.ProxyFromEnvironment,
 		}
 		return &http.Client{Transport: tr}
 	}
@@ -494,7 +492,6 @@ func (p *processor) usedAuthorizedClient() bool {
 
 // rolieFeedEntries loads the references to the advisory files for a given feed.
 func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
-
 	client := p.httpClient()
 	res, err := client.Get(feed)
 	p.badDirListings.use()
@@ -519,7 +516,7 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
 			return nil, nil, fmt.Errorf("%s: %v", feed, err)
 		}
 		var rolieDoc any
-		err = json.NewDecoder(bytes.NewReader(all)).Decode(&rolieDoc)
+		err = misc.StrictJSONParse(bytes.NewReader(all), &rolieDoc)
 		return rfeed, rolieDoc, err
 	}()
 	if err != nil {
@@ -537,7 +534,7 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
 	if len(errors) > 0 {
 		p.badProviderMetadata.error("%s: Validating against JSON schema failed:", feed)
 		for _, msg := range errors {
-			p.badProviderMetadata.error(strings.ReplaceAll(msg, `%`, `%%`))
+			p.badProviderMetadata.error("%s", strings.ReplaceAll(msg, `%`, `%%`))
 		}
 	}
 
@@ -545,10 +542,9 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
 	var files []csaf.AdvisoryFile
 
 	rfeed.Entries(func(entry *csaf.Entry) {
-
 		// Filter if we have date checking.
 		if accept := p.cfg.Range; accept != nil {
-			if pub := time.Time(entry.Published); !pub.IsZero() && !accept.Contains(pub) {
+			if t := time.Time(entry.Updated); !t.IsZero() && !accept.Contains(t) {
 				return
 			}
 		}
@@ -594,11 +590,17 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {
 
 		var file csaf.AdvisoryFile
 
-		if sha256 != "" || sha512 != "" || sign != "" {
+		switch {
-			file = csaf.HashedAdvisoryFile{url, sha256, sha512, sign}
+		case sha256 == "" && sha512 != "":
-		} else {
+			p.badROLIEFeed.info("%s has no sha256 hash file listed", url)
-			file = csaf.PlainAdvisoryFile(url)
+		case sha256 != "" && sha512 == "":
+			p.badROLIEFeed.info("%s has no sha512 hash file listed", url)
+		case sha256 == "" && sha512 == "":
+			p.badROLIEFeed.error("No hash listed on ROLIE feed %s", url)
+		case sign == "":
+			p.badROLIEFeed.error("No signature listed on ROLIE feed %s", url)
 		}
+		file = csaf.PlainAdvisoryFile{Path: url, SHA256: sha256, SHA512: sha512, Sign: sign}
 
 		files = append(files, file)
 	})
@@ -620,17 +622,13 @@ func makeAbsolute(base *url.URL) func(*url.URL) *url.URL {
 
 var yearFromURL = regexp.MustCompile(`.*/(\d{4})/[^/]+$`)
 
+// integrity checks several csaf.AdvisoryFiles for formal
+// mistakes, from conforming filenames to invalid advisories.
 func (p *processor) integrity(
 	files []csaf.AdvisoryFile,
-	base string,
 	mask whereType,
 	lg func(MessageType, string, ...any),
 ) error {
-	b, err := url.Parse(base)
-	if err != nil {
-		return err
-	}
-	makeAbs := makeAbsolute(b)
 	client := p.httpClient()
 
 	var data bytes.Buffer
@@ -641,9 +639,8 @@ func (p *processor) integrity(
 			lg(ErrorType, "Bad URL %s: %v", f, err)
 			continue
 		}
-		fp = makeAbs(fp)
 
-		u := b.ResolveReference(fp).String()
+		u := fp.String()
 
 		// Should this URL be ignored?
 		if p.cfg.ignoreURL(u) {
@@ -667,11 +664,6 @@ func (p *processor) integrity(
 		var folderYear *int
 		if m := yearFromURL.FindStringSubmatch(u); m != nil {
 			year, _ := strconv.Atoi(m[1])
-			// Check if the year is in the accepted time interval.
-			if accept := p.cfg.Range; accept != nil &&
-				!accept.Intersects(models.Year(year)) {
-				continue
-			}
 			folderYear = &year
 		}
 
@@ -686,9 +678,9 @@ func (p *processor) integrity(
 			continue
 		}
 
-		// Warn if we do not get JSON.
+		// Error if we do not get JSON.
 		if ct := res.Header.Get("Content-Type"); ct != "application/json" {
-			lg(WarnType,
+			lg(ErrorType,
 				"The content type of %s should be 'application/json' but is '%s'",
 				u, ct)
 		}
@@ -703,7 +695,7 @@ func (p *processor) integrity(
 		if err := func() error {
 			defer res.Body.Close()
 			tee := io.TeeReader(res.Body, hasher)
-			return json.NewDecoder(tee).Decode(&doc)
+			return misc.StrictJSONParse(tee, &doc)
 		}(); err != nil {
 			lg(ErrorType, "Reading %s failed: %v", u, err)
 			continue
@@ -740,50 +732,59 @@ func (p *processor) integrity(
 		// Check if file is in the right folder.
 		p.badFolders.use()
 
-		if date, err := p.expr.Eval(
+		switch date, fault := p.extractTime(doc, `initial_release_date`, u); {
-			`$.document.tracking.initial_release_date`, doc); err != nil {
+		case fault != "":
-			p.badFolders.error(
+			p.badFolders.error("%s", fault)
-				"Extracting 'initial_release_date' from %s failed: %v", u, err)
+		case folderYear == nil:
-		} else if text, ok := date.(string); !ok {
-			p.badFolders.error("'initial_release_date' is not a string in %s", u)
-		} else if d, err := time.Parse(time.RFC3339, text); err != nil {
-			p.badFolders.error(
-				"Parsing 'initial_release_date' as RFC3339 failed in %s: %v", u, err)
-		} else if folderYear == nil {
 			p.badFolders.error("No year folder found in %s", u)
-		} else if d.UTC().Year() != *folderYear {
+		case date.UTC().Year() != *folderYear:
-			p.badFolders.error("%s should be in folder %d", u, d.UTC().Year())
+			p.badFolders.error("%s should be in folder %d", u, date.UTC().Year())
+		}
+		current, fault := p.extractTime(doc, `current_release_date`, u)
+		if fault != "" {
+			p.badChanges.error("%s", fault)
+		} else {
+			p.timesAdv[f.URL()] = current
 		}
 
 		// Check hashes
 		p.badIntegrities.use()
 
-		for _, x := range []struct {
+		type hash struct {
 			ext  string
 			url  func() string
 			hash []byte
-		}{
+		}
-			{"SHA256", f.SHA256URL, s256.Sum(nil)},
+		hashes := []hash{}
-			{"SHA512", f.SHA512URL, s512.Sum(nil)},
+		if f.SHA256URL() != "" {
-		} {
+			hashes = append(hashes, hash{"SHA256", f.SHA256URL, s256.Sum(nil)})
+		}
+		if f.SHA512URL() != "" {
+			hashes = append(hashes, hash{"SHA512", f.SHA512URL, s512.Sum(nil)})
+		}
+
+		couldFetchHash := false
+		hashFetchErrors := []string{}
+
+		for _, x := range hashes {
 			hu, err := url.Parse(x.url())
 			if err != nil {
 				lg(ErrorType, "Bad URL %s: %v", x.url(), err)
 				continue
 			}
-			hu = makeAbs(hu)
+			hashFile := hu.String()
-			hashFile := b.ResolveReference(hu).String()
 
 			p.checkTLS(hashFile)
 			if res, err = client.Get(hashFile); err != nil {
-				p.badIntegrities.error("Fetching %s failed: %v.", hashFile, err)
+				hashFetchErrors = append(hashFetchErrors, fmt.Sprintf("Fetching %s failed: %v.", hashFile, err))
 				continue
 			}
 			if res.StatusCode != http.StatusOK {
-				p.badIntegrities.error("Fetching %s failed: Status code %d (%s)",
+				hashFetchErrors = append(hashFetchErrors, fmt.Sprintf("Fetching %s failed: Status code %d (%s)",
-					hashFile, res.StatusCode, res.Status)
+					hashFile, res.StatusCode, res.Status))
 				continue
 			}
+			couldFetchHash = true
 			h, err := func() ([]byte, error) {
 				defer res.Body.Close()
 				return util.HashFromReader(res.Body)
@@ -801,14 +802,26 @@ func (p *processor) integrity(
 					x.ext, u, hashFile)
 			}
 		}
+
+		msgType := ErrorType
+		// Log only as warning, if the other hash could be fetched
+		if couldFetchHash {
+			msgType = WarnType
+		}
+		if f.IsDirectory() {
+			msgType = InfoType
+		}
+		for _, fetchError := range hashFetchErrors {
+			p.badIntegrities.add(msgType, "%s", fetchError)
+		}
+
 		// Check signature
 		su, err := url.Parse(f.SignURL())
 		if err != nil {
 			lg(ErrorType, "Bad URL %s: %v", f.SignURL(), err)
 			continue
 		}
-		su = makeAbs(su)
+		sigFile := su.String()
-		sigFile := b.ResolveReference(su).String()
 		p.checkTLS(sigFile)
 
 		p.badSignatures.use()
@@ -846,9 +859,48 @@ func (p *processor) integrity(
 		}
 	}
 
+	// If we tested an existing changes.csv
+	if len(p.timesAdv) > 0 && p.badChanges.used() {
+		// Iterate over all files again
+		for _, f := range files {
+			// If there was no previous error when extracting times from advisories and we have a valid time
+			if timeAdv, ok := p.timesAdv[f.URL()]; ok {
+				// If there was no previous error when extracting times from changes and the file was listed in changes.csv
+				if timeCha, ok := p.timesChanges[f.URL()]; ok {
+					// check if the time matches
+					if !timeAdv.Equal(timeCha) {
+						// if not, give an error and remove the pair so it isn't reported multiple times should integrity be called again
+						p.badChanges.error("Current release date in changes.csv and %s is not identical.", f.URL())
+						delete(p.timesAdv, f.URL())
+						delete(p.timesChanges, f.URL())
+					}
+				}
+			}
+		}
+	}
+
 	return nil
 }
 
+// extractTime extracts a time.Time value from a json document and returns it and an empty string or zero time alongside
+// a string representing the error message that prevented obtaining the proper time value.
+func (p *processor) extractTime(doc any, value string, u any) (time.Time, string) {
+	filter := "$.document.tracking." + value
+	date, err := p.expr.Eval(filter, doc)
+	if err != nil {
+		return time.Time{}, fmt.Sprintf("Extracting '%s' from %s failed: %v", value, u, err)
+	}
+	text, ok := date.(string)
+	if !ok {
+		return time.Time{}, fmt.Sprintf("'%s' is not a string in %s", value, u)
+	}
+	d, err := time.Parse(time.RFC3339, text)
+	if err != nil {
+		return time.Time{}, fmt.Sprintf("Parsing '%s' as RFC3339 failed in %s: %v", value, u, err)
+	}
+	return d, ""
+}
+
 // checkIndex fetches the "index.txt" and calls "checkTLS" method for HTTPS checks.
 // It extracts the file names from the file and passes them to "integrity" function.
 // It returns error if fetching/reading the file(s) fails, otherwise nil.
@@ -889,11 +941,13 @@ func (p *processor) checkIndex(base string, mask whereType) error {
 		scanner := bufio.NewScanner(res.Body)
 		for line := 1; scanner.Scan(); line++ {
 			u := scanner.Text()
-			if _, err := url.Parse(u); err != nil {
+			up, err := url.Parse(u)
+			if err != nil {
 				p.badIntegrities.error("index.txt contains invalid URL %q in line %d", u, line)
 				continue
 			}
-			files = append(files, csaf.PlainAdvisoryFile(u))
+
+			files = append(files, csaf.DirectoryAdvisoryFile{Path: misc.JoinURL(bu, up).String()})
 		}
 		return files, scanner.Err()
 	}()
@@ -908,7 +962,7 @@ func (p *processor) checkIndex(base string, mask whereType) error {
 	// Block rolie checks.
 	p.labelChecker.feedLabel = ""
 
-	return p.integrity(files, base, mask, p.badIndices.add)
+	return p.integrity(files, mask, p.badIndices.add)
 }
 
 // checkChanges fetches the "changes.csv" and calls the "checkTLS" method for HTTPs checks.
@@ -916,7 +970,6 @@ func (p *processor) checkIndex(base string, mask whereType) error {
 // of the fields' values and if they are sorted properly. Then it passes the files to the
 // "integrity" functions. It returns error if some test fails, otherwise nil.
 func (p *processor) checkChanges(base string, mask whereType) error {
-
 	bu, err := url.Parse(base)
 	if err != nil {
 		return err
@@ -975,9 +1028,15 @@ func (p *processor) checkChanges(base string, mask whereType) error {
 				continue
 			}
 			path := r[pathColumn]
-			times, files =
+
-				append(times, t),
+			pathURL, err := url.Parse(path)
-				append(files, csaf.PlainAdvisoryFile(path))
+			if err != nil {
+				return nil, nil, err
+			}
+
+			times, files = append(times, t),
+				append(files, csaf.DirectoryAdvisoryFile{Path: misc.JoinURL(bu, pathURL).String()})
+			p.timesChanges[path] = t
 		}
 		return times, files, nil
 	}()
@@ -991,7 +1050,7 @@ func (p *processor) checkChanges(base string, mask whereType) error {
 		if p.cfg.Range != nil {
 			filtered = " (maybe filtered out by time interval)"
 		}
-		p.badChanges.warn("no entries in changes.csv found" + filtered)
+		p.badChanges.warn("%s", "no entries in changes.csv found"+filtered)
 	}
 
 	if !sort.SliceIsSorted(times, func(i, j int) bool {
@@ -1003,7 +1062,7 @@ func (p *processor) checkChanges(base string, mask whereType) error {
 	// Block rolie checks.
 	p.labelChecker.feedLabel = ""
 
-	return p.integrity(files, base, mask, p.badChanges.add)
+	return p.integrity(files, mask, p.badChanges.add)
 }
 
 // empty checks if list of strings contains at least one none empty string.
@@ -1149,7 +1208,6 @@ func (p *processor) checkMissing(string) error {
 // checkInvalid goes over all found adivisories URLs and checks
 // if file name conforms to standard.
 func (p *processor) checkInvalid(string) error {
-
 	p.badDirListings.use()
 	var invalids []string
 
@@ -1171,7 +1229,6 @@ func (p *processor) checkInvalid(string) error {
 // checkListing goes over all found adivisories URLs and checks
 // if their parent directory is listable.
 func (p *processor) checkListing(string) error {
-
 	p.badDirListings.use()
 
 	pgs := pages{}
@@ -1206,7 +1263,6 @@ func (p *processor) checkListing(string) error {
 // checkWhitePermissions checks if the TLP:WHITE advisories are
 // available with unprotected access.
 func (p *processor) checkWhitePermissions(string) error {
-
 	var ids []string
 	for id, open := range p.labelChecker.whiteAdvisories {
 		if !open {
@@ -1232,7 +1288,6 @@ func (p *processor) checkWhitePermissions(string) error {
 // According to the result, the respective error messages added to
 // badProviderMetadata.
 func (p *processor) checkProviderMetadata(domain string) bool {
-
 	p.badProviderMetadata.use()
 
 	client := p.httpClient()
@@ -1243,8 +1298,8 @@ func (p *processor) checkProviderMetadata(domain string) bool {
 
 	for i := range lpmd.Messages {
 		p.badProviderMetadata.warn(
-			"Unexpected situation while loading provider-metadata.json: " +
+			"Unexpected situation while loading provider-metadata.json: %s",
-				lpmd.Messages[i].Message)
+			lpmd.Messages[i].Message)
 	}
 
 	if !lpmd.Valid() {
@@ -1262,10 +1317,25 @@ func (p *processor) checkProviderMetadata(domain string) bool {
 // It checks the existence of the CSAF field in the file content and tries to fetch
 // the value of this field. Returns an empty string if no error was encountered,
 // the errormessage otherwise.
-func (p *processor) checkSecurity(domain string) string {
+func (p *processor) checkSecurity(domain string, legacy bool) (int, string) {
+	folder := "https://" + domain + "/"
+	if !legacy {
+		folder = folder + ".well-known/"
+	}
+	msg := p.checkSecurityFolder(folder)
+	if msg == "" {
+		if !legacy {
+			return 0, "Found valid security.txt within the well-known directory"
+		}
+		return 2, "Found valid security.txt in the legacy location"
+	}
+	return 1, folder + "security.txt: " + msg
+}
 
+// checkSecurityFolder checks the security.txt in a given folder.
+func (p *processor) checkSecurityFolder(folder string) string {
 	client := p.httpClient()
-	path := "https://" + domain + "/.well-known/security.txt"
+	path := folder + "security.txt"
 	res, err := client.Get(path)
 	if err != nil {
 		return fmt.Sprintf("Fetching %s failed: %v", path, err)
@@ -1293,17 +1363,11 @@ func (p *processor) checkSecurity(domain string) string {
 	}
 
 	// Try to load
-	up, err := url.Parse(u)
+	_, err = url.Parse(u)
 	if err != nil {
 		return fmt.Sprintf("CSAF URL '%s' invalid: %v", u, err)
 	}
 
-	base, err := url.Parse("https://" + domain + "/.well-known/")
-	if err != nil {
-		return err.Error()
-	}
-
-	u = base.ResolveReference(up).String()
 	p.checkTLS(u)
 	if res, err = client.Get(u); err != nil {
 		return fmt.Sprintf("Cannot fetch %s from security.txt: %v", u, err)
@@ -1328,50 +1392,52 @@ func (p *processor) checkSecurity(domain string) string {
 
 // checkDNS checks if the "csaf.data.security.domain.tld" DNS record is available
 // and serves the "provider-metadata.json".
-// It returns an empty string if all checks are passed, otherwise the errormessage.
+func (p *processor) checkDNS(domain string) {
-func (p *processor) checkDNS(domain string) string {
+	p.badDNSPath.use()
-
 	client := p.httpClient()
 	path := "https://csaf.data.security." + domain
 	res, err := client.Get(path)
 	if err != nil {
-		return fmt.Sprintf("Fetching %s failed: %v", path, err)
+		p.badDNSPath.add(ErrorType,
+			"Fetching %s failed: %v", path, err)
+		return
 	}
 	if res.StatusCode != http.StatusOK {
-		return fmt.Sprintf("Fetching %s failed. Status code %d (%s)",
+		p.badDNSPath.add(ErrorType, "Fetching %s failed. Status code %d (%s)",
 			path, res.StatusCode, res.Status)
-
 	}
 	hash := sha256.New()
 	defer res.Body.Close()
 	content, err := io.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Sprintf("Error while reading the response from %s", path)
+		p.badDNSPath.add(ErrorType,
+			"Error while reading the response from %s", path)
 	}
 	hash.Write(content)
 	if !bytes.Equal(hash.Sum(nil), p.pmd256) {
-		return fmt.Sprintf("%s does not serve the same provider-metadata.json as previously found", path)
+		p.badDNSPath.add(ErrorType,
+			"%s does not serve the same provider-metadata.json as previously found",
+			path)
 	}
-	return ""
 }
 
-// checkWellknownMetadataReporter checks if the provider-metadata.json file is
+// checkWellknown checks if the provider-metadata.json file is
-// available under the /.well-known/csaf/ directory. Returns the errormessage if
+// available under the /.well-known/csaf/ directory.
-// an error was encountered, or an empty string otherwise
+func (p *processor) checkWellknown(domain string) {
-func (p *processor) checkWellknown(domain string) string {
+	p.badWellknownMetadata.use()
-
 	client := p.httpClient()
 	path := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
 
 	res, err := client.Get(path)
 	if err != nil {
-		return fmt.Sprintf("Fetching %s failed: %v", path, err)
+		p.badWellknownMetadata.add(ErrorType,
+			"Fetching %s failed: %v", path, err)
+		return
 	}
 	if res.StatusCode != http.StatusOK {
-		return fmt.Sprintf("Fetching %s failed. Status code %d (%s)",
+		p.badWellknownMetadata.add(ErrorType, "Fetching %s failed. Status code %d (%s)",
 			path, res.StatusCode, res.Status)
 	}
-	return ""
 }
 
 // checkWellknownSecurityDNS
@@ -1383,46 +1449,49 @@ func (p *processor) checkWellknown(domain string) string {
 //  4. Finally it checks if the "csaf.data.security.domain.tld" DNS record
 //     is available and serves the "provider-metadata.json".
 //
-// /
+// For the security.txt checks, it first checks the default location.
-// If all three checks fail, errors are given,
+// Should this lookup fail, a warning is will be given and a lookup
-// otherwise warnings for all failed checks.
+// for the legacy location will be made. If this fails as well, then an
-// The function returns nil, unless errors outside the checks were found.
+// error is given.
-// In that case, errors are returned.
 func (p *processor) checkWellknownSecurityDNS(domain string) error {
+	p.checkWellknown(domain)
+	// Security check for well known (default) and legacy location
+	warnings, sDMessage := p.checkSecurity(domain, false)
+	// if the security.txt under .well-known was not okay
+	// check for a security.txt within its legacy location
+	sLMessage := ""
+	if warnings == 1 {
+		warnings, sLMessage = p.checkSecurity(domain, true)
+	}
 
-	warningsW := p.checkWellknown(domain)
-	warningsS := p.checkSecurity(domain)
-	warningsD := p.checkDNS(domain)
-
-	p.badWellknownMetadata.use()
 	p.badSecurity.use()
-	p.badDNSPath.use()
 
-	var kind MessageType
+	// Report about Securitytxt:
-	if warningsS == "" || warningsD == "" || warningsW == "" {
+	// Only report about default location if it was succesful (0).
-		kind = WarnType
+	// Report default and legacy as errors if neither was succesful (1).
-	} else {
+	// Warn about missing security in the default position if not found
-		kind = ErrorType
+	// but found in the legacy location, and inform about finding it there (2).
+	switch warnings {
+	case 0:
+		p.badSecurity.add(InfoType, "%s", sDMessage)
+	case 1:
+		p.badSecurity.add(ErrorType, "%s", sDMessage)
+		p.badSecurity.add(ErrorType, "%s", sLMessage)
+	case 2:
+		p.badSecurity.add(WarnType, "%s", sDMessage)
+		p.badSecurity.add(InfoType, "%s", sLMessage)
 	}
 
-	if warningsW != "" {
+	p.checkDNS(domain)
-		p.badWellknownMetadata.add(kind, warningsW)
+
-	}
-	if warningsS != "" {
-		p.badSecurity.add(kind, warningsS)
-	}
-	if warningsD != "" {
-		p.badDNSPath.add(kind, warningsD)
-	}
 	return nil
 }
 
 // checkPGPKeys checks if the OpenPGP keys are available and valid, fetches
-// the the remotely keys and compares the fingerprints.
+// the remote pubkeys and compares the fingerprints.
-// As a result of these a respective error messages are passed to badPGP method
+// As a result of these checks respective error messages are passed
-// in case of errors. It returns nil if all checks are passed.
+// to badPGP methods. It returns nil if all checks are passed.
 func (p *processor) checkPGPKeys(_ string) error {
-
 	p.badPGPs.use()
 
 	src, err := p.expr.Eval("$.public_openpgp_keys", p.pmd)
@@ -1446,11 +1515,6 @@ func (p *processor) checkPGPKeys(_ string) error {
 
 	client := p.httpClient()
 
-	base, err := url.Parse(p.pmdURL)
-	if err != nil {
-		return err
-	}
-
 	for i := range keys {
 		key := &keys[i]
 		if key.URL == nil {
@@ -1463,10 +1527,11 @@ func (p *processor) checkPGPKeys(_ string) error {
 			continue
 		}
 
-		u := base.ResolveReference(up).String()
+		// Todo: refactor all methods to directly accept *url.URL
+		u := up.String()
 		p.checkTLS(u)
 
-		res, err := client.Get(u)
+		res, err := client.Get(*key.URL)
 		if err != nil {
 			p.badPGPs.error("Fetching public OpenPGP key %s failed: %v.", u, err)
 			continue
@@ -1481,14 +1546,13 @@ func (p *processor) checkPGPKeys(_ string) error {
 			defer res.Body.Close()
 			return crypto.NewKeyFromArmoredReader(res.Body)
 		}()
-
 		if err != nil {
 			p.badPGPs.error("Reading public OpenPGP key %s failed: %v", u, err)
 			continue
 		}
 
 		if !strings.EqualFold(ckey.GetFingerprint(), string(key.Fingerprint)) {
-			p.badPGPs.error("Fingerprint of public OpenPGP key %s does not match remotely loaded.", u)
+			p.badPGPs.error("Given Fingerprint (%q) of public OpenPGP key %q does not match remotely loaded (%q).", string(key.Fingerprint), u, ckey.GetFingerprint())
 			continue
 		}
 		if p.keys == nil {

cmd/csaf_checker/processor_test.go

@@ -0,0 +1,259 @@
+// This file is Free Software under the Apache-2.0 License
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http/httptest"
+	"os"
+	"reflect"
+	"slices"
+	"strings"
+	"testing"
+	"text/template"
+
+	"github.com/gocsaf/csaf/v3/internal/testutil"
+	"github.com/gocsaf/csaf/v3/util"
+)
+
+func getRequirementTestData(t *testing.T, params testutil.ProviderParams, directoryProvider bool) []Requirement {
+	path := "../../testdata/processor-requirements/"
+	if params.EnableSha256 {
+		path += "sha256-"
+	}
+	if params.EnableSha512 {
+		path += "sha512-"
+	}
+	if params.ForbidSha256 {
+		path += "forbid-sha256-"
+	}
+	if params.ForbidSha512 {
+		path += "forbid-sha512-"
+	}
+	if directoryProvider {
+		path += "directory"
+	} else {
+		path += "rolie"
+	}
+	path += ".json"
+
+	content, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmplt, err := template.New("base").Parse(string(content))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var output bytes.Buffer
+	err = tmplt.Execute(&output, params)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var requirement []Requirement
+	err = json.Unmarshal(output.Bytes(), &requirement)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return requirement
+}
+
+func TestContentTypeReport(t *testing.T) {
+	serverURL := ""
+	params := testutil.ProviderParams{
+		URL:             "",
+		EnableSha256:    true,
+		EnableSha512:    true,
+		ForbidSha256:    true,
+		ForbidSha512:    true,
+		JSONContentType: "application/json; charset=utf-8",
+	}
+	server := httptest.NewTLSServer(testutil.ProviderHandler(&params, false))
+	defer server.Close()
+
+	serverURL = server.URL
+	params.URL = server.URL
+
+	hClient := server.Client()
+	client := util.Client(hClient)
+
+	cfg := config{}
+	err := cfg.prepare()
+	if err != nil {
+		t.Fatalf("SHA marking config failed: %v", err)
+	}
+	p, err := newProcessor(&cfg)
+	if err != nil {
+		t.Fatalf("could not init downloader: %v", err)
+	}
+	p.client = client
+
+	report, err := p.run([]string{serverURL + "/provider-metadata.json"})
+	if err != nil {
+		t.Errorf("Content-Type-Report: Expected no error, got: %v", err)
+	}
+
+	got := report.Domains[0].Requirements
+	idx := slices.IndexFunc(got, func(e *Requirement) bool {
+		return e.Num == 7
+	})
+	if idx == -1 {
+		t.Error("Content-Type-Report: Could not find requirement")
+	} else {
+		message := got[idx].Messages[0]
+		if message.Type != ErrorType || !strings.Contains(message.Text, "should be 'application/json'") {
+			t.Errorf("Content-Type-Report: Content Type Error, got %v", message)
+		}
+	}
+
+	p.close()
+}
+
+func TestShaMarking(t *testing.T) {
+	tests := []struct {
+		name              string
+		directoryProvider bool
+		enableSha256      bool
+		enableSha512      bool
+		forbidSha256      bool
+		forbidSha512      bool
+	}{
+		{
+			name:              "deliver sha256 and sha512",
+			directoryProvider: false,
+			enableSha256:      true,
+			enableSha512:      true,
+		},
+		{
+			name:              "enable sha256 and sha512, forbid fetching",
+			directoryProvider: false,
+			enableSha256:      true,
+			enableSha512:      true,
+			forbidSha256:      true,
+			forbidSha512:      true,
+		},
+		{
+			name:              "enable sha256 and sha512, forbid sha256",
+			directoryProvider: false,
+			enableSha256:      true,
+			enableSha512:      true,
+			forbidSha256:      true,
+			forbidSha512:      false,
+		},
+		{
+			name:              "enable sha256 and sha512, forbid sha512",
+			directoryProvider: false,
+			enableSha256:      true,
+			enableSha512:      true,
+			forbidSha256:      false,
+			forbidSha512:      true,
+		},
+		{
+			name:              "only deliver sha256",
+			directoryProvider: false,
+			enableSha256:      true,
+			enableSha512:      false,
+		},
+		{
+			name:              "only deliver sha512",
+			directoryProvider: false,
+			enableSha256:      false,
+			enableSha512:      true,
+		},
+		{
+			name:              "deliver sha256 and sha512, directory provider",
+			directoryProvider: true,
+			enableSha256:      true,
+			enableSha512:      true,
+		},
+		{
+			name:              "only deliver sha256, directory provider",
+			directoryProvider: true,
+			enableSha256:      true,
+			enableSha512:      false,
+		},
+		{
+			name:              "only deliver sha512, directory provider",
+			directoryProvider: true,
+			enableSha256:      false,
+			enableSha512:      true,
+		},
+		{
+			name:              "no hash",
+			directoryProvider: false,
+			enableSha256:      false,
+			enableSha512:      false,
+		},
+		{
+			name:              "no hash, directory provider",
+			directoryProvider: true,
+			enableSha256:      false,
+			enableSha512:      false,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			serverURL := ""
+			params := testutil.ProviderParams{
+				URL:          "",
+				EnableSha256: test.enableSha256,
+				EnableSha512: test.enableSha512,
+				ForbidSha256: test.forbidSha256,
+				ForbidSha512: test.forbidSha512,
+			}
+			server := httptest.NewTLSServer(testutil.ProviderHandler(&params, test.directoryProvider))
+			defer server.Close()
+
+			serverURL = server.URL
+			params.URL = server.URL
+
+			hClient := server.Client()
+			client := util.Client(hClient)
+
+			cfg := config{}
+			err := cfg.prepare()
+			if err != nil {
+				t.Fatalf("SHA marking config failed: %v", err)
+			}
+			p, err := newProcessor(&cfg)
+			if err != nil {
+				t.Fatalf("could not init downloader: %v", err)
+			}
+			p.client = client
+
+			report, err := p.run([]string{serverURL + "/provider-metadata.json"})
+			if err != nil {
+				t.Errorf("SHA marking %v: Expected no error, got: %v", test.name, err)
+			}
+			expected := getRequirementTestData(t,
+				testutil.ProviderParams{
+					URL:          serverURL,
+					EnableSha256: test.enableSha256,
+					EnableSha512: test.enableSha512,
+					ForbidSha256: test.forbidSha256,
+					ForbidSha512: test.forbidSha512,
+				},
+				test.directoryProvider)
+			for i, got := range report.Domains[0].Requirements {
+				if !reflect.DeepEqual(expected[i], *got) {
+					t.Errorf("SHA marking %v: Expected %v, got %v", test.name, expected[i], *got)
+				}
+			}
+
+			p.close()
+		})
+	}
+}

cmd/csaf_checker/report.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -18,8 +18,8 @@ import (
 	"os"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/models"
+	"github.com/gocsaf/csaf/v3/internal/models"
 )
 
 // MessageType is the kind of the message.

cmd/csaf_checker/reporters.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -13,7 +13,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 type (
@@ -178,7 +178,7 @@ func (r *tlpAmberRedReporter) report(p *processor, domain *Domain) {
 		return
 	}
 	if len(p.badAmberRedPermissions) == 0 {
-		req.message(InfoType, "All tested advisories labeled TLP:WHITE or TLP:RED were access-protected.")
+		req.message(InfoType, "All tested advisories labeled TLP:AMBER or TLP:RED were access-protected.")
 		return
 	}
 	req.Messages = p.badAmberRedPermissions
@@ -251,10 +251,6 @@ func (r *securityReporter) report(p *processor, domain *Domain) {
 		req.message(WarnType, "Performed no in-depth test of security.txt.")
 		return
 	}
-	if len(p.badSecurity) == 0 {
-		req.message(InfoType, "Found CSAF entry in security.txt.")
-		return
-	}
 	req.Messages = p.badSecurity
 }
 

cmd/csaf_checker/roliecheck.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -15,8 +15,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // identifier consist of document/tracking/id and document/publisher/namespace,
@@ -216,11 +216,6 @@ func defaults[T any](p *T, def T) T {
 // processROLIEFeeds goes through all ROLIE feeds and checks their
 // integrity and completeness.
 func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
-
-	base, err := url.Parse(p.pmdURL)
-	if err != nil {
-		return err
-	}
 	p.badROLIEFeed.use()
 
 	advisories := map[*csaf.Feed][]csaf.AdvisoryFile{}
@@ -232,12 +227,11 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 			if feed.URL == nil {
 				continue
 			}
-			up, err := url.Parse(string(*feed.URL))
+			feedBase, err := url.Parse(string(*feed.URL))
 			if err != nil {
 				p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err)
 				continue
 			}
-			feedBase := base.ResolveReference(up)
 			feedURL := feedBase.String()
 			p.checkTLS(feedURL)
 
@@ -264,13 +258,12 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 				continue
 			}
 
-			up, err := url.Parse(string(*feed.URL))
+			feedURL, err := url.Parse(string(*feed.URL))
 			if err != nil {
 				p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err)
 				continue
 			}
 
-			feedURL := base.ResolveReference(up)
 			feedBase, err := util.BaseURL(feedURL)
 			if err != nil {
 				p.badProviderMetadata.error("Bad base path: %v", err)
@@ -290,7 +283,7 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 			// TODO: Issue a warning if we want check AMBER+ without an
 			// authorizing client.
 
-			if err := p.integrity(files, feedBase, rolieMask, p.badProviderMetadata.add); err != nil {
+			if err := p.integrity(files, rolieMask, p.badProviderMetadata.add); err != nil {
 				if err != errContinue {
 					return err
 				}
@@ -319,13 +312,12 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 				continue
 			}
 
-			up, err := url.Parse(string(*feed.URL))
+			feedBase, err := url.Parse(string(*feed.URL))
 			if err != nil {
 				p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err)
 				continue
 			}
 
-			feedBase := base.ResolveReference(up)
 			makeAbs := makeAbsolute(feedBase)
 			label := defaults(feed.TLPLabel, csaf.TLPLabelUnlabeled)
 

cmd/csaf_checker/rules.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -12,7 +12,7 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
 )
 
 type ruleCondition int

cmd/csaf_downloader/config.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -19,10 +19,10 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
+	"github.com/gocsaf/csaf/v3/internal/certs"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/filter"
+	"github.com/gocsaf/csaf/v3/internal/filter"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/models"
+	"github.com/gocsaf/csaf/v3/internal/models"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
 )
 
 const (
@@ -41,6 +41,13 @@ const (
 	validationUnsafe = validationMode("unsafe")
 )
 
+type hashAlgorithm string
+
+const (
+	algSha256 = hashAlgorithm("sha256")
+	algSha512 = hashAlgorithm("sha512")
+)
+
 type config struct {
 	Directory            string            `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"`
 	Insecure             bool              `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"`
@@ -57,6 +64,8 @@ type config struct {
 	IgnorePattern        []string          `long:"ignore_pattern" short:"i" description:"Do not download files if their URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignore_pattern"`
 	ExtraHeader          http.Header       `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"`
 
+	EnumeratePMDOnly bool `long:"enumerate_pmd_only" description:"If this flag is set to true, the downloader will only enumerate valid provider metadata files, but not download documents" toml:"enumerate_pmd_only"`
+
 	RemoteValidator        string   `long:"validator" description:"URL to validate documents remotely" value-name:"URL" toml:"validator"`
 	RemoteValidatorCache   string   `long:"validator_cache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validator_cache"`
 	RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"`
@@ -77,6 +86,9 @@ type config struct {
 
 	clientCerts   []tls.Certificate
 	ignorePattern filter.PatternMatcher
+
+	//lint:ignore SA5008 We are using choice or than once: sha256, sha512
+	PreferredHash hashAlgorithm `long:"preferred_hash" choice:"sha256" choice:"sha512" value-name:"HASH" description:"HASH to prefer" toml:"preferred_hash"`
 }
 
 // configPaths are the potential file locations of the config file.
@@ -218,7 +230,7 @@ func (cfg *config) prepareLogging() error {
 		w = f
 	}
 	ho := slog.HandlerOptions{
-		//AddSource: true,
+		// AddSource: true,
 		Level:       cfg.LogLevel.Level,
 		ReplaceAttr: dropSubSeconds,
 	}

cmd/csaf_downloader/downloader.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022, 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022, 2023 Intevation GmbH <https://intevation.de>
@@ -25,6 +25,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -33,14 +34,22 @@ import (
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 	"golang.org/x/time/rate"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
+type hashFetchInfo struct {
+	url       string
+	preferred bool
+	warn      bool
+	hashType  hashAlgorithm
+}
+
 type downloader struct {
 	cfg       *config
+	client    *util.Client // Used for testing
 	keys      *crypto.KeyRing
-	eval      *util.PathEval
 	validator csaf.RemoteValidator
 	forwarder *forwarder
 	mkdirMu   sync.Mutex
@@ -54,7 +63,6 @@ type downloader struct {
 const failedValidationDir = "failed_validation"
 
 func newDownloader(cfg *config) (*downloader, error) {
-
 	var validator csaf.RemoteValidator
 
 	if cfg.RemoteValidator != "" {
@@ -73,7 +81,6 @@ func newDownloader(cfg *config) (*downloader, error) {
 
 	return &downloader{
 		cfg:       cfg,
-		eval:      util.NewPathEval(),
 		validator: validator,
 	}, nil
 }
@@ -105,7 +112,6 @@ func logRedirect(req *http.Request, via []*http.Request) error {
 }
 
 func (d *downloader) httpClient() util.Client {
-
 	hClient := http.Client{}
 
 	if d.cfg.verbose() {
@@ -123,16 +129,20 @@ func (d *downloader) httpClient() util.Client {
 
 	hClient.Transport = &http.Transport{
 		TLSClientConfig: &tlsConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
 
 	client := util.Client(&hClient)
 
+	// Overwrite for testing purposes
+	if d.client != nil {
+		client = *d.client
+	}
+
 	// Add extra headers.
-	if len(d.cfg.ExtraHeader) > 0 {
+	client = &util.HeaderClient{
-		client = &util.HeaderClient{
+		Client: client,
-			Client: client,
+		Header: d.cfg.ExtraHeader,
-			Header: d.cfg.ExtraHeader,
-		}
 	}
 
 	// Add optional URL logging.
@@ -164,6 +174,36 @@ func httpLog(who string) func(string, string) {
 	}
 }
 
+func (d *downloader) enumerate(domain string) error {
+	client := d.httpClient()
+
+	loader := csaf.NewProviderMetadataLoader(client)
+	lpmd := loader.Enumerate(domain)
+
+	docs := []any{}
+
+	for _, pmd := range lpmd {
+		if d.cfg.verbose() {
+			for i := range pmd.Messages {
+				slog.Debug("Enumerating provider-metadata.json",
+					"domain", domain,
+					"message", pmd.Messages[i].Message)
+			}
+		}
+
+		docs = append(docs, pmd.Document)
+	}
+
+	// print the results
+	doc, err := json.MarshalIndent(docs, "", "  ")
+	if err != nil {
+		slog.Error("Couldn't marshal PMD document json")
+	}
+	fmt.Println(string(doc))
+
+	return nil
+}
+
 func (d *downloader) download(ctx context.Context, domain string) error {
 	client := d.httpClient()
 
@@ -171,7 +211,14 @@ func (d *downloader) download(ctx context.Context, domain string) error {
 
 	lpmd := loader.Load(domain)
 
-	if d.cfg.verbose() {
+	if !lpmd.Valid() {
+		for i := range lpmd.Messages {
+			slog.Error("Loading provider-metadata.json",
+				"domain", domain,
+				"message", lpmd.Messages[i].Message)
+		}
+		return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain)
+	} else if d.cfg.verbose() {
 		for i := range lpmd.Messages {
 			slog.Debug("Loading provider-metadata.json",
 				"domain", domain,
@@ -179,28 +226,26 @@ func (d *downloader) download(ctx context.Context, domain string) error {
 		}
 	}
 
-	if !lpmd.Valid() {
+	pmdURL, err := url.Parse(lpmd.URL)
-		return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain)
-	}
-
-	base, err := url.Parse(lpmd.URL)
 	if err != nil {
 		return fmt.Errorf("invalid URL '%s': %v", lpmd.URL, err)
 	}
 
+	expr := util.NewPathEval()
+
 	if err := d.loadOpenPGPKeys(
 		client,
 		lpmd.Document,
-		base,
+		expr,
 	); err != nil {
 		return err
 	}
 
 	afp := csaf.NewAdvisoryFileProcessor(
 		client,
-		d.eval,
+		expr,
 		lpmd.Document,
-		base)
+		pmdURL)
 
 	// Do we need time range based filtering?
 	if d.cfg.Range != nil {
@@ -219,7 +264,6 @@ func (d *downloader) downloadFiles(
 	label csaf.TLPLabel,
 	files []csaf.AdvisoryFile,
 ) error {
-
 	var (
 		advisoryCh = make(chan csaf.AdvisoryFile)
 		errorCh    = make(chan error)
@@ -266,10 +310,9 @@ allFiles:
 func (d *downloader) loadOpenPGPKeys(
 	client util.Client,
 	doc any,
-	base *url.URL,
+	expr *util.PathEval,
 ) error {
-
+	src, err := expr.Eval("$.public_openpgp_keys", doc)
-	src, err := d.eval.Eval("$.public_openpgp_keys", doc)
 	if err != nil {
 		// no keys.
 		return nil
@@ -291,7 +334,7 @@ func (d *downloader) loadOpenPGPKeys(
 		if key.URL == nil {
 			continue
 		}
-		up, err := url.Parse(*key.URL)
+		u, err := url.Parse(*key.URL)
 		if err != nil {
 			slog.Warn("Invalid URL",
 				"url", *key.URL,
@@ -299,9 +342,7 @@ func (d *downloader) loadOpenPGPKeys(
 			continue
 		}
 
-		u := base.ResolveReference(up).String()
+		res, err := client.Get(u.String())
-
-		res, err := client.Get(u)
 		if err != nil {
 			slog.Warn(
 				"Fetching public OpenPGP key failed",
@@ -322,7 +363,6 @@ func (d *downloader) loadOpenPGPKeys(
 			defer res.Body.Close()
 			return crypto.NewKeyFromArmoredReader(res.Body)
 		}()
-
 		if err != nil {
 			slog.Warn(
 				"Reading public OpenPGP key failed",
@@ -334,7 +374,7 @@ func (d *downloader) loadOpenPGPKeys(
 		if !strings.EqualFold(ckey.GetFingerprint(), string(key.Fingerprint)) {
 			slog.Warn(
 				"Fingerprint of public OpenPGP key does not match remotely loaded",
-				"url", u)
+				"url", u, "fingerprint", key.Fingerprint, "remote-fingerprint", ckey.GetFingerprint())
 			continue
 		}
 		if d.keys == nil {
@@ -374,6 +414,321 @@ func (d *downloader) logValidationIssues(url string, errors []string, err error)
 	}
 }
 
+// downloadContext stores the common context of a downloader.
+type downloadContext struct {
+	d                  *downloader
+	client             util.Client
+	data               bytes.Buffer
+	lastDir            string
+	initialReleaseDate time.Time
+	dateExtract        func(any) error
+	lower              string
+	stats              stats
+	expr               *util.PathEval
+}
+
+func newDownloadContext(d *downloader, label csaf.TLPLabel) *downloadContext {
+	dc := &downloadContext{
+		d:      d,
+		client: d.httpClient(),
+		lower:  strings.ToLower(string(label)),
+		expr:   util.NewPathEval(),
+	}
+	dc.dateExtract = util.TimeMatcher(&dc.initialReleaseDate, time.RFC3339)
+	return dc
+}
+
+func (dc *downloadContext) downloadAdvisory(
+	file csaf.AdvisoryFile,
+	errorCh chan<- error,
+) error {
+	u, err := url.Parse(file.URL())
+	if err != nil {
+		dc.stats.downloadFailed++
+		slog.Warn("Ignoring invalid URL",
+			"url", file.URL(),
+			"error", err)
+		return nil
+	}
+
+	if dc.d.cfg.ignoreURL(file.URL()) {
+		slog.Debug("Ignoring URL", "url", file.URL())
+		return nil
+	}
+
+	// Ignore not conforming filenames.
+	filename := filepath.Base(u.Path)
+	if !util.ConformingFileName(filename) {
+		dc.stats.filenameFailed++
+		slog.Warn("Ignoring none conforming filename",
+			"filename", filename)
+		return nil
+	}
+
+	resp, err := dc.client.Get(file.URL())
+	if err != nil {
+		dc.stats.downloadFailed++
+		slog.Warn("Cannot GET",
+			"url", file.URL(),
+			"error", err)
+		return nil
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		dc.stats.downloadFailed++
+		slog.Warn("Cannot load",
+			"url", file.URL(),
+			"status", resp.Status,
+			"status_code", resp.StatusCode)
+		return nil
+	}
+
+	// Warn if we do not get JSON.
+	if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
+		slog.Warn("Content type is not 'application/json'",
+			"url", file.URL(),
+			"content_type", ct)
+	}
+
+	var (
+		writers                    []io.Writer
+		s256, s512                 hash.Hash
+		s256Data, s512Data         []byte
+		remoteSHA256, remoteSHA512 []byte
+		signData                   []byte
+	)
+
+	hashToFetch := []hashFetchInfo{}
+	if file.SHA512URL() != "" {
+		hashToFetch = append(hashToFetch, hashFetchInfo{
+			url:       file.SHA512URL(),
+			warn:      true,
+			hashType:  algSha512,
+			preferred: strings.EqualFold(string(dc.d.cfg.PreferredHash), string(algSha512)),
+		})
+	} else {
+		slog.Info("SHA512 not present")
+	}
+	if file.SHA256URL() != "" {
+		hashToFetch = append(hashToFetch, hashFetchInfo{
+			url:       file.SHA256URL(),
+			warn:      true,
+			hashType:  algSha256,
+			preferred: strings.EqualFold(string(dc.d.cfg.PreferredHash), string(algSha256)),
+		})
+	} else {
+		slog.Info("SHA256 not present")
+	}
+	if file.IsDirectory() {
+		for i := range hashToFetch {
+			hashToFetch[i].warn = false
+		}
+	}
+
+	remoteSHA256, s256Data, remoteSHA512, s512Data = loadHashes(dc.client, hashToFetch)
+	if remoteSHA512 != nil {
+		s512 = sha512.New()
+		writers = append(writers, s512)
+	}
+	if remoteSHA256 != nil {
+		s256 = sha256.New()
+		writers = append(writers, s256)
+	}
+
+	// Remember the data as we need to store it to file later.
+	dc.data.Reset()
+	writers = append(writers, &dc.data)
+
+	// Download the advisory and hash it.
+	hasher := io.MultiWriter(writers...)
+
+	var doc any
+
+	tee := io.TeeReader(resp.Body, hasher)
+
+	if err := misc.StrictJSONParse(tee, &doc); err != nil {
+		dc.stats.downloadFailed++
+		slog.Warn("Downloading failed",
+			"url", file.URL(),
+			"error", err)
+		return nil
+	}
+
+	// Compare the checksums.
+	s256Check := func() error {
+		if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) {
+			dc.stats.sha256Failed++
+			return fmt.Errorf("SHA256 checksum of %s does not match", file.URL())
+		}
+		return nil
+	}
+
+	s512Check := func() error {
+		if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) {
+			dc.stats.sha512Failed++
+			return fmt.Errorf("SHA512 checksum of %s does not match", file.URL())
+		}
+		return nil
+	}
+
+	// Validate OpenPGP signature.
+	keysCheck := func() error {
+		// Only check signature if we have loaded keys.
+		if dc.d.keys == nil {
+			return nil
+		}
+		var sign *crypto.PGPSignature
+		sign, signData, err = loadSignature(dc.client, file.SignURL())
+		if err != nil {
+			slog.Warn("Downloading signature failed",
+				"url", file.SignURL(),
+				"error", err)
+		}
+		if sign != nil {
+			if err := dc.d.checkSignature(dc.data.Bytes(), sign); err != nil {
+				if !dc.d.cfg.IgnoreSignatureCheck {
+					dc.stats.signatureFailed++
+					return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err)
+				}
+			}
+		}
+		return nil
+	}
+
+	// Validate against CSAF schema.
+	schemaCheck := func() error {
+		if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 {
+			dc.stats.schemaFailed++
+			dc.d.logValidationIssues(file.URL(), errors, err)
+			return fmt.Errorf("schema validation for %q failed", file.URL())
+		}
+		return nil
+	}
+
+	// Validate if filename is conforming.
+	filenameCheck := func() error {
+		if err := util.IDMatchesFilename(dc.expr, doc, filename); err != nil {
+			dc.stats.filenameFailed++
+			return fmt.Errorf("filename not conforming %s: %s", file.URL(), err)
+		}
+		return nil
+	}
+
+	// Validate against remote validator.
+	remoteValidatorCheck := func() error {
+		if dc.d.validator == nil {
+			return nil
+		}
+		rvr, err := dc.d.validator.Validate(doc)
+		if err != nil {
+			errorCh <- fmt.Errorf(
+				"calling remote validator on %q failed: %w",
+				file.URL(), err)
+			return nil
+		}
+		if !rvr.Valid {
+			dc.stats.remoteFailed++
+			return fmt.Errorf("remote validation of %q failed", file.URL())
+		}
+		return nil
+	}
+
+	// Run all the validations.
+	valStatus := notValidatedValidationStatus
+	for _, check := range []func() error{
+		s256Check,
+		s512Check,
+		keysCheck,
+		schemaCheck,
+		filenameCheck,
+		remoteValidatorCheck,
+	} {
+		if err := check(); err != nil {
+			slog.Error("Validation check failed", "error", err)
+			valStatus.update(invalidValidationStatus)
+			if dc.d.cfg.ValidationMode == validationStrict {
+				return nil
+			}
+		}
+	}
+	valStatus.update(validValidationStatus)
+
+	// Send to forwarder
+	if dc.d.forwarder != nil {
+		dc.d.forwarder.forward(
+			filename, dc.data.String(),
+			valStatus,
+			string(s256Data),
+			string(s512Data))
+	}
+
+	if dc.d.cfg.NoStore {
+		// Do not write locally.
+		if valStatus == validValidationStatus {
+			dc.stats.succeeded++
+		}
+		return nil
+	}
+
+	if err := dc.expr.Extract(
+		`$.document.tracking.initial_release_date`, dc.dateExtract, false, doc,
+	); err != nil {
+		slog.Warn("Cannot extract initial_release_date from advisory",
+			"url", file.URL())
+		dc.initialReleaseDate = time.Now()
+	}
+	dc.initialReleaseDate = dc.initialReleaseDate.UTC()
+
+	// Advisories that failed validation are stored in a special folder.
+	var newDir string
+	if valStatus != validValidationStatus {
+		newDir = path.Join(dc.d.cfg.Directory, failedValidationDir)
+	} else {
+		newDir = dc.d.cfg.Directory
+	}
+
+	// Do we have a configured destination folder?
+	if dc.d.cfg.Folder != "" {
+		newDir = path.Join(newDir, dc.d.cfg.Folder)
+	} else {
+		newDir = path.Join(newDir, dc.lower, strconv.Itoa(dc.initialReleaseDate.Year()))
+	}
+
+	if newDir != dc.lastDir {
+		if err := dc.d.mkdirAll(newDir, 0755); err != nil {
+			errorCh <- err
+			return nil
+		}
+		dc.lastDir = newDir
+	}
+
+	// Write advisory to file
+	path := filepath.Join(dc.lastDir, filename)
+
+	// Write data to disk.
+	for _, x := range []struct {
+		p string
+		d []byte
+	}{
+		{path, dc.data.Bytes()},
+		{path + ".sha256", s256Data},
+		{path + ".sha512", s512Data},
+		{path + ".asc", signData},
+	} {
+		if x.d != nil {
+			if err := os.WriteFile(x.p, x.d, 0644); err != nil {
+				errorCh <- err
+				return nil
+			}
+		}
+	}
+
+	dc.stats.succeeded++
+	slog.Info("Written advisory", "path", path)
+	return nil
+}
+
 func (d *downloader) downloadWorker(
 	ctx context.Context,
 	wg *sync.WaitGroup,
@@ -383,20 +738,11 @@ func (d *downloader) downloadWorker(
 ) {
 	defer wg.Done()
 
-	var (
+	dc := newDownloadContext(d, label)
-		client             = d.httpClient()
-		data               bytes.Buffer
-		lastDir            string
-		initialReleaseDate time.Time
-		dateExtract        = util.TimeMatcher(&initialReleaseDate, time.RFC3339)
-		lower              = strings.ToLower(string(label))
-		stats              = stats{}
-	)
 
 	// Add collected stats back to total.
-	defer d.addStats(&stats)
+	defer d.addStats(&dc.stats)
 
-nextAdvisory:
 	for {
 		var file csaf.AdvisoryFile
 		var ok bool
@@ -408,272 +754,10 @@ nextAdvisory:
 		case <-ctx.Done():
 			return
 		}
-
+		if err := dc.downloadAdvisory(file, errorCh); err != nil {
-		u, err := url.Parse(file.URL())
+			slog.Error("download terminated", "error", err)
-		if err != nil {
+			return
-			stats.downloadFailed++
-			slog.Warn("Ignoring invalid URL",
-				"url", file.URL(),
-				"error", err)
-			continue
 		}
-
-		if d.cfg.ignoreURL(file.URL()) {
-			slog.Debug("Ignoring URL", "url", file.URL())
-			continue
-		}
-
-		// Ignore not conforming filenames.
-		filename := filepath.Base(u.Path)
-		if !util.ConformingFileName(filename) {
-			stats.filenameFailed++
-			slog.Warn("Ignoring none conforming filename",
-				"filename", filename)
-			continue
-		}
-
-		resp, err := client.Get(file.URL())
-		if err != nil {
-			stats.downloadFailed++
-			slog.Warn("Cannot GET",
-				"url", file.URL(),
-				"error", err)
-			continue
-		}
-
-		if resp.StatusCode != http.StatusOK {
-			stats.downloadFailed++
-			slog.Warn("Cannot load",
-				"url", file.URL(),
-				"status", resp.Status,
-				"status_code", resp.StatusCode)
-			continue
-		}
-
-		// Warn if we do not get JSON.
-		if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
-			slog.Warn("Content type is not 'application/json'",
-				"url", file.URL(),
-				"content_type", ct)
-		}
-
-		var (
-			writers                    []io.Writer
-			s256, s512                 hash.Hash
-			s256Data, s512Data         []byte
-			remoteSHA256, remoteSHA512 []byte
-			signData                   []byte
-		)
-
-		// Only hash when we have a remote counter part we can compare it with.
-		if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
-			slog.Warn("Cannot fetch SHA256",
-				"url", file.SHA256URL(),
-				"error", err)
-		} else {
-			s256 = sha256.New()
-			writers = append(writers, s256)
-		}
-
-		if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
-			slog.Warn("Cannot fetch SHA512",
-				"url", file.SHA512URL(),
-				"error", err)
-		} else {
-			s512 = sha512.New()
-			writers = append(writers, s512)
-		}
-
-		// Remember the data as we need to store it to file later.
-		data.Reset()
-		writers = append(writers, &data)
-
-		// Download the advisory and hash it.
-		hasher := io.MultiWriter(writers...)
-
-		var doc any
-
-		if err := func() error {
-			defer resp.Body.Close()
-			tee := io.TeeReader(resp.Body, hasher)
-			return json.NewDecoder(tee).Decode(&doc)
-		}(); err != nil {
-			stats.downloadFailed++
-			slog.Warn("Downloading failed",
-				"url", file.URL(),
-				"error", err)
-			continue
-		}
-
-		// Compare the checksums.
-		s256Check := func() error {
-			if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) {
-				stats.sha256Failed++
-				return fmt.Errorf("SHA256 checksum of %s does not match", file.URL())
-			}
-			return nil
-		}
-
-		s512Check := func() error {
-			if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) {
-				stats.sha512Failed++
-				return fmt.Errorf("SHA512 checksum of %s does not match", file.URL())
-			}
-			return nil
-		}
-
-		// Validate OpenPGP signature.
-		keysCheck := func() error {
-			// Only check signature if we have loaded keys.
-			if d.keys == nil {
-				return nil
-			}
-			var sign *crypto.PGPSignature
-			sign, signData, err = loadSignature(client, file.SignURL())
-			if err != nil {
-				slog.Warn("Downloading signature failed",
-					"url", file.SignURL(),
-					"error", err)
-			}
-			if sign != nil {
-				if err := d.checkSignature(data.Bytes(), sign); err != nil {
-					if !d.cfg.IgnoreSignatureCheck {
-						stats.signatureFailed++
-						return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err)
-					}
-				}
-			}
-			return nil
-		}
-
-		// Validate against CSAF schema.
-		schemaCheck := func() error {
-			if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 {
-				stats.schemaFailed++
-				d.logValidationIssues(file.URL(), errors, err)
-				return fmt.Errorf("schema validation for %q failed", file.URL())
-			}
-			return nil
-		}
-
-		// Validate if filename is conforming.
-		filenameCheck := func() error {
-			if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil {
-				stats.filenameFailed++
-				return fmt.Errorf("filename not conforming %s: %s", file.URL(), err)
-			}
-			return nil
-		}
-
-		// Validate against remote validator.
-		remoteValidatorCheck := func() error {
-			if d.validator == nil {
-				return nil
-			}
-			rvr, err := d.validator.Validate(doc)
-			if err != nil {
-				errorCh <- fmt.Errorf(
-					"calling remote validator on %q failed: %w",
-					file.URL(), err)
-				return nil
-			}
-			if !rvr.Valid {
-				stats.remoteFailed++
-				return fmt.Errorf("remote validation of %q failed", file.URL())
-			}
-			return nil
-		}
-
-		// Run all the validations.
-		valStatus := notValidatedValidationStatus
-		for _, check := range []func() error{
-			s256Check,
-			s512Check,
-			keysCheck,
-			schemaCheck,
-			filenameCheck,
-			remoteValidatorCheck,
-		} {
-			if err := check(); err != nil {
-				slog.Error("Validation check failed", "error", err)
-				valStatus.update(invalidValidationStatus)
-				if d.cfg.ValidationMode == validationStrict {
-					continue nextAdvisory
-				}
-			}
-		}
-		valStatus.update(validValidationStatus)
-
-		// Send to forwarder
-		if d.forwarder != nil {
-			d.forwarder.forward(
-				filename, data.String(),
-				valStatus,
-				string(s256Data),
-				string(s512Data))
-		}
-
-		if d.cfg.NoStore {
-			// Do not write locally.
-			if valStatus == validValidationStatus {
-				stats.succeeded++
-			}
-			continue
-		}
-
-		if err := d.eval.Extract(`$.document.tracking.initial_release_date`, dateExtract, false, doc); err != nil {
-			slog.Warn("Cannot extract initial_release_date from advisory",
-				"url", file.URL())
-			initialReleaseDate = time.Now()
-		}
-		initialReleaseDate = initialReleaseDate.UTC()
-
-		// Advisories that failed validation are store in a special folder.
-		var newDir string
-		if valStatus != validValidationStatus {
-			newDir = path.Join(d.cfg.Directory, failedValidationDir, lower)
-		} else {
-			newDir = path.Join(d.cfg.Directory, lower)
-		}
-
-		// Do we have a configured destination folder?
-		if d.cfg.Folder != "" {
-			newDir = path.Join(newDir, d.cfg.Folder)
-		} else {
-			newDir = path.Join(newDir, strconv.Itoa(initialReleaseDate.Year()))
-		}
-
-		if newDir != lastDir {
-			if err := d.mkdirAll(newDir, 0755); err != nil {
-				errorCh <- err
-				continue
-			}
-			lastDir = newDir
-		}
-
-		// Write advisory to file
-		path := filepath.Join(lastDir, filename)
-
-		// Write data to disk.
-		for _, x := range []struct {
-			p string
-			d []byte
-		}{
-			{path, data.Bytes()},
-			{path + ".sha256", s256Data},
-			{path + ".sha512", s512Data},
-			{path + ".asc", signData},
-		} {
-			if x.d != nil {
-				if err := os.WriteFile(x.p, x.d, 0644); err != nil {
-					errorCh <- err
-					continue nextAdvisory
-				}
-			}
-		}
-
-		stats.succeeded++
-		slog.Info("Written advisory", "path", path)
 	}
 }
 
@@ -694,11 +778,11 @@ func loadSignature(client util.Client, p string) (*crypto.PGPSignature, []byte,
 	if err != nil {
 		return nil, nil, err
 	}
+	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		return nil, nil, fmt.Errorf(
 			"fetching signature from '%s' failed: %s (%d)", p, resp.Status, resp.StatusCode)
 	}
-	defer resp.Body.Close()
 	data, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, nil, err
@@ -710,16 +794,60 @@ func loadSignature(client util.Client, p string) (*crypto.PGPSignature, []byte,
 	return sign, data, nil
 }
 
+func loadHashes(client util.Client, hashes []hashFetchInfo) ([]byte, []byte, []byte, []byte) {
+	var remoteSha256, remoteSha512, sha256Data, sha512Data []byte
+
+	// Load preferred hashes first
+	slices.SortStableFunc(hashes, func(a, b hashFetchInfo) int {
+		if a.preferred == b.preferred {
+			return 0
+		}
+		if a.preferred && !b.preferred {
+			return -1
+		}
+		return 1
+	})
+	for _, h := range hashes {
+		if remote, data, err := loadHash(client, h.url); err != nil {
+			if h.warn {
+				slog.Warn("Cannot fetch hash",
+					"hash", h.hashType,
+					"url", h.url,
+					"error", err)
+			} else {
+				slog.Info("Hash not present", "hash", h.hashType, "file", h.url)
+			}
+		} else {
+			switch h.hashType {
+			case algSha512:
+				{
+					remoteSha512 = remote
+					sha512Data = data
+				}
+			case algSha256:
+				{
+					remoteSha256 = remote
+					sha256Data = data
+				}
+			}
+			if h.preferred {
+				break
+			}
+		}
+	}
+	return remoteSha256, sha256Data, remoteSha512, sha512Data
+}
+
 func loadHash(client util.Client, p string) ([]byte, []byte, error) {
 	resp, err := client.Get(p)
 	if err != nil {
 		return nil, nil, err
 	}
+	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		return nil, nil, fmt.Errorf(
 			"fetching hash from '%s' failed: %s (%d)", p, resp.Status, resp.StatusCode)
 	}
-	defer resp.Body.Close()
 	var data bytes.Buffer
 	tee := io.TeeReader(resp.Body, &data)
 	hash, err := util.HashFromReader(tee)
@@ -739,3 +867,14 @@ func (d *downloader) run(ctx context.Context, domains []string) error {
 	}
 	return nil
 }
+
+// runEnumerate performs the enumeration of PMDs for all the given domains.
+func (d *downloader) runEnumerate(domains []string) error {
+	defer d.stats.log()
+	for _, domain := range domains {
+		if err := d.enumerate(domain); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/csaf_downloader/downloader_test.go

@@ -0,0 +1,160 @@
+// This file is Free Software under the Apache-2.0 License
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
+
+package main
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/gocsaf/csaf/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/testutil"
+	"github.com/gocsaf/csaf/v3/util"
+)
+
+func checkIfFileExists(path string, t *testing.T) bool {
+	if _, err := os.Stat(path); err == nil {
+		return true
+	} else if !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("Failed to check if file exists: %v", err)
+	}
+	return false
+}
+
+func TestShaMarking(t *testing.T) {
+	tests := []struct {
+		name              string
+		directoryProvider bool
+		wantSha256        bool
+		wantSha512        bool
+		enableSha256      bool
+		enableSha512      bool
+		preferredHash     hashAlgorithm
+	}{
+		{
+			name:              "want sha256 and sha512",
+			directoryProvider: false,
+			wantSha256:        true,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+		},
+		{
+			name:              "only want sha256",
+			directoryProvider: false,
+			wantSha256:        true,
+			wantSha512:        false,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha256,
+		},
+		{
+			name:              "only want sha512",
+			directoryProvider: false,
+			wantSha256:        false,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha512,
+		},
+		{
+			name:              "only want sha512",
+			directoryProvider: false,
+			wantSha256:        false,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha512,
+		},
+
+		{
+			name:              "only deliver sha256",
+			directoryProvider: false,
+			wantSha256:        true,
+			wantSha512:        false,
+			enableSha256:      true,
+			enableSha512:      false,
+			preferredHash:     algSha512,
+		},
+		{
+			name:              "only want sha256, directory provider",
+			directoryProvider: true,
+			wantSha256:        true,
+			wantSha512:        false,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha256,
+		},
+		{
+			name:              "only want sha512, directory provider",
+			directoryProvider: true,
+			wantSha256:        false,
+			wantSha512:        true,
+			enableSha256:      true,
+			enableSha512:      true,
+			preferredHash:     algSha512,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			serverURL := ""
+			params := testutil.ProviderParams{
+				URL:          "",
+				EnableSha256: test.enableSha256,
+				EnableSha512: test.enableSha512,
+			}
+			server := httptest.NewTLSServer(testutil.ProviderHandler(&params, test.directoryProvider))
+			defer server.Close()
+
+			serverURL = server.URL
+			params.URL = server.URL
+
+			hClient := server.Client()
+			client := util.Client(hClient)
+
+			tempDir := t.TempDir()
+			cfg := config{LogLevel: &options.LogLevel{Level: slog.LevelDebug}, Directory: tempDir, PreferredHash: test.preferredHash}
+			err := cfg.prepare()
+			if err != nil {
+				t.Fatalf("SHA marking config failed: %v", err)
+			}
+			d, err := newDownloader(&cfg)
+			if err != nil {
+				t.Fatalf("could not init downloader: %v", err)
+			}
+			d.client = &client
+
+			ctx := context.Background()
+			err = d.run(ctx, []string{serverURL + "/provider-metadata.json"})
+			if err != nil {
+				t.Errorf("SHA marking %v: Expected no error, got: %v", test.name, err)
+			}
+			d.close()
+
+			// Check for downloaded hashes
+			sha256Exists := checkIfFileExists(tempDir+"/white/2020/avendor-advisory-0004.json.sha256", t)
+			sha512Exists := checkIfFileExists(tempDir+"/white/2020/avendor-advisory-0004.json.sha512", t)
+
+			if sha256Exists != test.wantSha256 {
+				t.Errorf("%v: expected sha256 hash present to be %v, got: %v", test.name, test.wantSha256, sha256Exists)
+			}
+
+			if sha512Exists != test.wantSha512 {
+				t.Errorf("%v: expected sha512 hash present to be %v, got: %v", test.name, test.wantSha512, sha512Exists)
+			}
+		})
+	}
+}

cmd/csaf_downloader/forwarder.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -19,8 +19,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/internal/misc"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // failedForwardDir is the name of the special sub folder
@@ -57,7 +57,10 @@ type forwarder struct {
 
 // newForwarder creates a new forwarder.
 func newForwarder(cfg *config) *forwarder {
-	queue := max(1, cfg.ForwardQueue)
+	queue := cfg.ForwardQueue
+	if queue < 1 {
+		queue = 1
+	}
 	return &forwarder{
 		cfg:  cfg,
 		cmds: make(chan func(*forwarder), queue),
@@ -103,16 +106,15 @@ func (f *forwarder) httpClient() util.Client {
 
 	hClient.Transport = &http.Transport{
 		TLSClientConfig: &tlsConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
 
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(f.cfg.ForwardHeader) > 0 {
+	client = &util.HeaderClient{
-		client = &util.HeaderClient{
+		Client: client,
-			Client: client,
+		Header: f.cfg.ForwardHeader,
-			Header: f.cfg.ForwardHeader,
-		}
 	}
 
 	// Add optional URL logging.
@@ -222,12 +224,12 @@ func (f *forwarder) storeFailed(filename, doc, sha256, sha512 string) {
 
 // limitedString reads max bytes from reader and returns it as a string.
 // Longer strings are indicated by "..." as a suffix.
-func limitedString(r io.Reader, max int) (string, error) {
+func limitedString(r io.Reader, maxLength int) (string, error) {
 	var msg strings.Builder
-	if _, err := io.Copy(&msg, io.LimitReader(r, int64(max))); err != nil {
+	if _, err := io.Copy(&msg, io.LimitReader(r, int64(maxLength))); err != nil {
 		return "", err
 	}
-	if msg.Len() >= max {
+	if msg.Len() >= maxLength {
 		msg.WriteString("...")
 	}
 	return msg.String(), nil

cmd/csaf_downloader/forwarder_test.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -23,8 +23,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 func TestValidationStatusUpdate(t *testing.T) {

cmd/csaf_downloader/main.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -15,7 +15,7 @@ import (
 	"os"
 	"os/signal"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
 )
 
 func run(cfg *config, domains []string) error {
@@ -40,6 +40,11 @@ func run(cfg *config, domains []string) error {
 		d.forwarder = f
 	}
 
+	// If the enumerate-only flag is set, enumerate found PMDs,
+	// else use the normal load method
+	if cfg.EnumeratePMDOnly {
+		return d.runEnumerate(domains)
+	}
 	return d.run(ctx, domains)
 }
 

cmd/csaf_downloader/stats.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>

cmd/csaf_downloader/stats_test.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>

cmd/csaf_provider/actions.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -26,8 +26,8 @@ import (
 	"github.com/ProtonMail/gopenpgp/v2/constants"
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 const dateFormat = time.RFC3339

cmd/csaf_provider/config.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -11,6 +11,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"strings"
 
@@ -18,7 +19,7 @@ import (
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 	"golang.org/x/crypto/bcrypt"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
 )
 
 const (
@@ -262,6 +263,14 @@ func loadConfig() (*config, error) {
 	if cfg.CanonicalURLPrefix == "" {
 		cfg.CanonicalURLPrefix = "https://" + os.Getenv("SERVER_NAME")
 	}
+	// Check if canonical url prefix is invalid
+	parsedURL, err := url.ParseRequestURI(cfg.CanonicalURLPrefix)
+	if err != nil {
+		return nil, err
+	}
+	if parsedURL.Scheme != "https" && parsedURL.Scheme != "http" {
+		return nil, fmt.Errorf("invalid canonical URL: %q", cfg.CanonicalURLPrefix)
+	}
 
 	if cfg.TLPs == nil {
 		cfg.TLPs = []tlp{tlpCSAF, tlpWhite, tlpGreen, tlpAmber, tlpRed}

cmd/csaf_provider/controller.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -174,7 +174,7 @@ func (c *controller) web(
 // writeJSON sets the header for the response and writes the JSON encoding of the given "content".
 // It logs out an error message in case of an error.
 func writeJSON(rw http.ResponseWriter, content any, code int) {
-	rw.Header().Set("Content-type", "application/json; charset=utf-8")
+	rw.Header().Set("Content-type", "application/json")
 	rw.Header().Set("X-Content-Type-Options", "nosniff")
 	rw.WriteHeader(code)
 	if err := json.NewEncoder(rw).Encode(content); err != nil {

cmd/csaf_provider/create.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -22,8 +22,8 @@ import (
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // ensureFolders initializes the paths and call functions to create

cmd/csaf_provider/files.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -13,7 +13,7 @@ import (
 	"crypto/sha512"
 	"os"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 func writeHashedFile(fname, name string, data []byte, armored string) error {

cmd/csaf_provider/indices.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -18,7 +18,7 @@ import (
 	"sort"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 func updateIndex(dir, fname string) error {

cmd/csaf_provider/main.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -18,7 +18,7 @@ import (
 
 	"github.com/jessevdk/go-flags"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 type options struct {
@@ -48,7 +48,7 @@ func main() {
 
 	cfg, err := loadConfig()
 	if err != nil {
-		cgi.Serve(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		cgi.Serve(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
 			http.Error(rw, "Something went wrong. Check server logs for more details",
 				http.StatusInternalServerError)
 		}))

cmd/csaf_provider/mux.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>

cmd/csaf_provider/rolie.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -15,8 +15,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // mergeCategories merges the given categories into the old ones.

cmd/csaf_provider/tmpl/create.html

@@ -1,8 +1,8 @@
 <!--
- This file is Free Software under the MIT License
+ This file is Free Software under the Apache-2.0 License
- without warranty, see README.md and LICENSES/MIT.txt for details.
+ without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 
- SPDX-License-Identifier: MIT
+ SPDX-License-Identifier: Apache-2.0
 
  SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
  Software-Engineering: 2021 Intevation GmbH <https://intevation.de>

cmd/csaf_provider/tmpl/index.html

@@ -1,8 +1,8 @@
 <!--
- This file is Free Software under the MIT License
+ This file is Free Software under the Apache-2.0 License
- without warranty, see README.md and LICENSES/MIT.txt for details.
+ without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 
- SPDX-License-Identifier: MIT
+ SPDX-License-Identifier: Apache-2.0
 
  SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
  Software-Engineering: 2021 Intevation GmbH <https://intevation.de>

cmd/csaf_provider/tmpl/upload.html

@@ -1,8 +1,8 @@
 <!--
- This file is Free Software under the MIT License
+ This file is Free Software under the Apache-2.0 License
- without warranty, see README.md and LICENSES/MIT.txt for details.
+ without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 
- SPDX-License-Identifier: MIT
+ SPDX-License-Identifier: Apache-2.0
 
  SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
  Software-Engineering: 2021 Intevation GmbH <https://intevation.de>

cmd/csaf_provider/transaction.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -12,8 +12,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 func doTransaction(

cmd/csaf_uploader/config.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -18,8 +18,8 @@ import (
 	"golang.org/x/crypto/bcrypt"
 	"golang.org/x/term"
 
-	"github.com/csaf-poc/csaf_distribution/v3/internal/certs"
+	"github.com/gocsaf/csaf/v3/internal/certs"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/options"
+	"github.com/gocsaf/csaf/v3/internal/options"
 )
 
 const (

cmd/csaf_uploader/main.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -9,7 +9,7 @@
 // Implements a command line tool that uploads csaf documents to csaf_provider.
 package main
 
-import "github.com/csaf-poc/csaf_distribution/v3/internal/options"
+import "github.com/gocsaf/csaf/v3/internal/options"
 
 func main() {
 	args, cfg, err := parseArgsConfig()

cmd/csaf_uploader/processor.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022, 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022, 2023 Intevation GmbH <https://intevation.de>
@@ -11,7 +11,6 @@ package main
 import (
 	"bytes"
 	"crypto/tls"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -26,9 +25,9 @@ import (
 	"github.com/ProtonMail/gopenpgp/v2/constants"
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/internal/misc"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 type processor struct {
@@ -51,6 +50,7 @@ func (p *processor) httpClient() *http.Client {
 
 	client.Transport = &http.Transport{
 		TLSClientConfig: &tlsConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
 
 	return &client
@@ -81,8 +81,9 @@ func (p *processor) create() error {
 	}
 	defer resp.Body.Close()
 
+	var createError error
 	if resp.StatusCode != http.StatusOK {
-		log.Printf("Create failed: %s\n", resp.Status)
+		createError = fmt.Errorf("create failed: %s", resp.Status)
 	}
 
 	var result struct {
@@ -90,7 +91,7 @@ func (p *processor) create() error {
 		Errors  []string `json:"errors"`
 	}
 
-	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+	if err := misc.StrictJSONParse(resp.Body, &result); err != nil {
 		return err
 	}
 
@@ -100,7 +101,7 @@ func (p *processor) create() error {
 
 	writeStrings("Errors:", result.Errors)
 
-	return nil
+	return createError
 }
 
 // uploadRequest creates the request for uploading a csaf document by passing the filename.
@@ -114,7 +115,7 @@ func (p *processor) uploadRequest(filename string) (*http.Request, error) {
 
 	if !p.cfg.NoSchemaCheck {
 		var doc any
-		if err := json.NewDecoder(bytes.NewReader(data)).Decode(&doc); err != nil {
+		if err := misc.StrictJSONParse(bytes.NewReader(data), &doc); err != nil {
 			return nil, err
 		}
 		errs, err := csaf.ValidateCSAF(doc)
@@ -238,7 +239,7 @@ func (p *processor) process(filename string) error {
 		Errors      []string `json:"errors"`
 	}
 
-	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+	if err := misc.StrictJSONParse(resp.Body, &result); err != nil {
 		return err
 	}
 

cmd/csaf_validator/main.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -10,7 +10,6 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -18,8 +17,16 @@ import (
 
 	"github.com/jessevdk/go-flags"
 
-	"github.com/csaf-poc/csaf_distribution/v3/csaf"
+	"github.com/gocsaf/csaf/v3/csaf"
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
+)
+
+const (
+	exitCodeSchemaInvalid = 2 << iota
+	exitCodeNoRemoteValidator
+	exitCodeFailedRemoteValidation
+	exitCodeAllValid = 0
 )
 
 type options struct {
@@ -53,6 +60,7 @@ func main() {
 
 // run validates the given files.
 func run(opts *options, files []string) error {
+	exitCode := exitCodeAllValid
 
 	var validator csaf.RemoteValidator
 	eval := util.NewPathEval()
@@ -69,6 +77,9 @@ func run(opts *options, files []string) error {
 				"preparing remote validator failed: %w", err)
 		}
 		defer validator.Close()
+	} else {
+		exitCode |= exitCodeNoRemoteValidator
+		log.Printf("warn: no remote validator specified")
 	}
 
 	// Select amount level of output for remote validation.
@@ -96,7 +107,7 @@ func run(opts *options, files []string) error {
 			log.Printf("error: loading %q as JSON failed: %v\n", file, err)
 			continue
 		}
-		// Validate agsinst Schema.
+		// Validate against Schema.
 		validationErrs, err := csaf.ValidateCSAF(doc)
 		if err != nil {
 			log.Printf("error: validating %q against schema failed: %v\n",
@@ -104,6 +115,7 @@ func run(opts *options, files []string) error {
 
 		}
 		if len(validationErrs) > 0 {
+			exitCode |= exitCodeSchemaInvalid
 			fmt.Printf("schema validation errors of %q\n", file)
 			for _, vErr := range validationErrs {
 				fmt.Printf("  * %s\n", vErr)
@@ -112,10 +124,9 @@ func run(opts *options, files []string) error {
 			fmt.Printf("%q passes the schema validation.\n", file)
 		}
 
-		// Check filename agains ID
+		// Check filename against ID
 		if err := util.IDMatchesFilename(eval, doc, filepath.Base(file)); err != nil {
 			log.Printf("%s: %s.\n", file, err)
-			continue
 		}
 
 		// Validate against remote validator.
@@ -130,12 +141,15 @@ func run(opts *options, files []string) error {
 			if rvr.Valid {
 				passes = "passes"
 			} else {
+				exitCode |= exitCodeFailedRemoteValidation
 				passes = "does not pass"
 			}
 			fmt.Printf("%q %s remote validation.\n", file, passes)
 		}
 	}
 
+	// Exit code is based on validation results
+	os.Exit(exitCode)
 	return nil
 }
 
@@ -287,7 +301,7 @@ func loadJSONFromFile(fname string) (any, error) {
 	}
 	defer f.Close()
 	var doc any
-	if err = json.NewDecoder(f).Decode(&doc); err != nil {
+	if err = misc.StrictJSONParse(f, &doc); err != nil {
 		return nil, err
 	}
 	return doc, err

csaf/advisories.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -9,92 +9,109 @@
 package csaf
 
 import (
+	"context"
 	"encoding/csv"
 	"fmt"
 	"io"
-	"log"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // AdvisoryFile constructs the urls of a remote file.
 type AdvisoryFile interface {
+	slog.LogValuer
 	URL() string
 	SHA256URL() string
 	SHA512URL() string
 	SignURL() string
+	IsDirectory() bool
 }
 
-// PlainAdvisoryFile is a simple implementation of checkFile.
+// PlainAdvisoryFile contains all relevant urls of a remote file.
+type PlainAdvisoryFile struct {
+	Path   string
+	SHA256 string
+	SHA512 string
+	Sign   string
+}
+
+// URL returns the URL of this advisory.
+func (paf PlainAdvisoryFile) URL() string { return paf.Path }
+
+// SHA256URL returns the URL of SHA256 hash file of this advisory.
+func (paf PlainAdvisoryFile) SHA256URL() string { return paf.SHA256 }
+
+// SHA512URL returns the URL of SHA512 hash file of this advisory.
+func (paf PlainAdvisoryFile) SHA512URL() string { return paf.SHA512 }
+
+// SignURL returns the URL of signature file of this advisory.
+func (paf PlainAdvisoryFile) SignURL() string { return paf.Sign }
+
+// IsDirectory returns true, if was fetched via directory feeds.
+func (paf PlainAdvisoryFile) IsDirectory() bool { return false }
+
+// LogValue implements [slog.LogValuer]
+func (paf PlainAdvisoryFile) LogValue() slog.Value {
+	return slog.GroupValue(slog.String("url", paf.URL()))
+}
+
+// DirectoryAdvisoryFile only contains the base file path.
 // The hash and signature files are directly constructed by extending
 // the file name.
-type PlainAdvisoryFile string
+type DirectoryAdvisoryFile struct {
-
+	Path string
-// URL returns the URL of this advisory.
-func (paf PlainAdvisoryFile) URL() string { return string(paf) }
-
-// SHA256URL returns the URL of SHA256 hash file of this advisory.
-func (paf PlainAdvisoryFile) SHA256URL() string { return string(paf) + ".sha256" }
-
-// SHA512URL returns the URL of SHA512 hash file of this advisory.
-func (paf PlainAdvisoryFile) SHA512URL() string { return string(paf) + ".sha512" }
-
-// SignURL returns the URL of signature file of this advisory.
-func (paf PlainAdvisoryFile) SignURL() string { return string(paf) + ".asc" }
-
-// HashedAdvisoryFile is a more involed version of checkFile.
-// Here each component can be given explicitly.
-// If a component is not given it is constructed by
-// extending the first component.
-type HashedAdvisoryFile [4]string
-
-func (haf HashedAdvisoryFile) name(i int, ext string) string {
-	if haf[i] != "" {
-		return haf[i]
-	}
-	return haf[0] + ext
 }
 
 // URL returns the URL of this advisory.
-func (haf HashedAdvisoryFile) URL() string { return haf[0] }
+func (daf DirectoryAdvisoryFile) URL() string { return daf.Path }
 
 // SHA256URL returns the URL of SHA256 hash file of this advisory.
-func (haf HashedAdvisoryFile) SHA256URL() string { return haf.name(1, ".sha256") }
+func (daf DirectoryAdvisoryFile) SHA256URL() string { return daf.Path + ".sha256" }
 
 // SHA512URL returns the URL of SHA512 hash file of this advisory.
-func (haf HashedAdvisoryFile) SHA512URL() string { return haf.name(2, ".sha512") }
+func (daf DirectoryAdvisoryFile) SHA512URL() string { return daf.Path + ".sha512" }
 
 // SignURL returns the URL of signature file of this advisory.
-func (haf HashedAdvisoryFile) SignURL() string { return haf.name(3, ".asc") }
+func (daf DirectoryAdvisoryFile) SignURL() string { return daf.Path + ".asc" }
+
+// IsDirectory returns true, if was fetched via directory feeds.
+func (daf DirectoryAdvisoryFile) IsDirectory() bool { return true }
+
+// LogValue implements [slog.LogValuer]
+func (daf DirectoryAdvisoryFile) LogValue() slog.Value {
+	return slog.GroupValue(slog.String("url", daf.URL()))
+}
 
 // AdvisoryFileProcessor implements the extraction of
 // advisory file names from a given provider metadata.
 type AdvisoryFileProcessor struct {
 	AgeAccept func(time.Time) bool
-	Log       func(format string, args ...any)
+	Log       func(loglevel slog.Level, format string, args ...any)
 	client    util.Client
 	expr      *util.PathEval
 	doc       any
-	base      *url.URL
+	pmdURL    *url.URL
 }
 
-// NewAdvisoryFileProcessor constructs an filename extractor
+// NewAdvisoryFileProcessor constructs a filename extractor
 // for a given metadata document.
 func NewAdvisoryFileProcessor(
 	client util.Client,
 	expr *util.PathEval,
 	doc any,
-	base *url.URL,
+	pmdURL *url.URL,
 ) *AdvisoryFileProcessor {
 	return &AdvisoryFileProcessor{
 		client: client,
 		expr:   expr,
 		doc:    doc,
-		base:   base,
+		pmdURL: pmdURL,
 	}
 }
 
@@ -108,15 +125,15 @@ func empty(arr []string) bool {
 	return true
 }
 
-// Process extracts the adivisory filenames and passes them with
+// Process extracts the advisory filenames and passes them with
 // the corresponding label to fn.
 func (afp *AdvisoryFileProcessor) Process(
 	fn func(TLPLabel, []AdvisoryFile) error,
 ) error {
 	lg := afp.Log
 	if lg == nil {
-		lg = func(format string, args ...any) {
+		lg = func(loglevel slog.Level, format string, args ...any) {
-			log.Printf("AdvisoryFileProcessor.Process: "+format, args...)
+			slog.Log(context.Background(), loglevel, "AdvisoryFileProcessor.Process: "+format, args...)
 		}
 	}
 
@@ -124,7 +141,7 @@ func (afp *AdvisoryFileProcessor) Process(
 	rolie, err := afp.expr.Eval(
 		"$.distributions[*].rolie.feeds", afp.doc)
 	if err != nil {
-		lg("rolie check failed: %v\n", err)
+		lg(slog.LevelError, "rolie check failed", "err", err)
 		return err
 	}
 
@@ -136,7 +153,7 @@ func (afp *AdvisoryFileProcessor) Process(
 		if err := util.ReMarshalJSON(&feeds, rolie); err != nil {
 			return err
 		}
-		lg("Found %d ROLIE feed(s).\n", len(feeds))
+		lg(slog.LevelInfo, "Found ROLIE feed(s)", "length", len(feeds))
 
 		for _, feed := range feeds {
 			if err := afp.processROLIE(feed, fn); err != nil {
@@ -152,18 +169,18 @@ func (afp *AdvisoryFileProcessor) Process(
 		var dirURLs []string
 
 		if err != nil {
-			lg("extracting directory URLs failed: %v\n", err)
+			lg(slog.LevelError, "extracting directory URLs failed", "err", err)
 		} else {
 			var ok bool
 			dirURLs, ok = util.AsStrings(directoryURLs)
 			if !ok {
-				lg("directory_urls are not strings.\n")
+				lg(slog.LevelError, "directory_urls are not strings")
 			}
 		}
 
 		// Not found -> fall back to PMD url
 		if empty(dirURLs) {
-			baseURL, err := util.BaseURL(afp.base)
+			baseURL, err := util.BaseURL(afp.pmdURL)
 			if err != nil {
 				return err
 			}
@@ -193,9 +210,8 @@ func (afp *AdvisoryFileProcessor) Process(
 // prefixed by baseURL/.
 func (afp *AdvisoryFileProcessor) loadChanges(
 	baseURL string,
-	lg func(string, ...any),
+	lg func(slog.Level, string, ...any),
 ) ([]AdvisoryFile, error) {
-
 	base, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
@@ -228,12 +244,12 @@ func (afp *AdvisoryFileProcessor) loadChanges(
 			return nil, err
 		}
 		if len(r) < 2 {
-			lg("%q has not enough columns in line %d", line)
+			lg(slog.LevelError, "Not enough columns", "line", line)
 			continue
 		}
 		t, err := time.Parse(time.RFC3339, r[timeColumn])
 		if err != nil {
-			lg("%q has an invalid time stamp in line %d: %v", changesURL, line, err)
+			lg(slog.LevelError, "Invalid time stamp in line", "url", changesURL, "line", line, "err", err)
 			continue
 		}
 		// Apply date range filtering.
@@ -242,11 +258,17 @@ func (afp *AdvisoryFileProcessor) loadChanges(
 		}
 		path := r[pathColumn]
 		if _, err := url.Parse(path); err != nil {
-			lg("%q contains an invalid URL %q in line %d", changesURL, path, line)
+			lg(slog.LevelError, "Contains an invalid URL", "url", changesURL, "path", path, "line", line)
 			continue
 		}
+
+		pathURL, err := url.Parse(path)
+		if err != nil {
+			return nil, err
+		}
+
 		files = append(files,
-			PlainAdvisoryFile(base.JoinPath(path).String()))
+			DirectoryAdvisoryFile{Path: misc.JoinURL(base, pathURL).String()})
 	}
 	return files, nil
 }
@@ -260,33 +282,27 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 		if feed.URL == nil {
 			continue
 		}
-		up, err := url.Parse(string(*feed.URL))
+		feedURL, err := url.Parse(string(*feed.URL))
 		if err != nil {
-			log.Printf("Invalid URL %s in feed: %v.", *feed.URL, err)
+			slog.Error("Invalid URL in feed", "feed", *feed.URL, "err", err)
 			continue
 		}
-		feedURL := afp.base.ResolveReference(up)
+		slog.Info("Got feed URL", "feed", feedURL)
-		log.Printf("Feed URL: %s\n", feedURL)
 
 		fb, err := util.BaseURL(feedURL)
 		if err != nil {
-			log.Printf("error: Invalid feed base URL '%s': %v\n", fb, err)
+			slog.Error("Invalid feed base URL", "url", fb, "err", err)
-			continue
-		}
-		feedBaseURL, err := url.Parse(fb)
-		if err != nil {
-			log.Printf("error: Cannot parse feed base URL '%s': %v\n", fb, err)
 			continue
 		}
 
 		res, err := afp.client.Get(feedURL.String())
 		if err != nil {
-			log.Printf("error: Cannot get feed '%s'\n", err)
+			slog.Error("Cannot get feed", "err", err)
 			continue
 		}
 		if res.StatusCode != http.StatusOK {
-			log.Printf("error: Fetching %s failed. Status code %d (%s)",
+			slog.Error("Fetching failed",
-				feedURL, res.StatusCode, res.Status)
+				"url", feedURL, "status_code", res.StatusCode, "status", res.Status)
 			continue
 		}
 		rfeed, err := func() (*ROLIEFeed, error) {
@@ -294,7 +310,7 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 			return LoadROLIEFeed(res.Body)
 		}()
 		if err != nil {
-			log.Printf("Loading ROLIE feed failed: %v.", err)
+			slog.Error("Loading ROLIE feed failed", "err", err)
 			continue
 		}
 
@@ -306,17 +322,16 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 			}
 			p, err := url.Parse(u)
 			if err != nil {
-				log.Printf("error: Invalid URL '%s': %v", u, err)
+				slog.Error("Invalid URL", "url", u, "err", err)
 				return ""
 			}
-			return feedBaseURL.ResolveReference(p).String()
+			return p.String()
 		}
 
 		rfeed.Entries(func(entry *Entry) {
-
 			// Filter if we have date checking.
 			if afp.AgeAccept != nil {
-				if pub := time.Time(entry.Published); !pub.IsZero() && !afp.AgeAccept(pub) {
+				if t := time.Time(entry.Updated); !t.IsZero() && !afp.AgeAccept(t) {
 					return
 				}
 			}
@@ -347,10 +362,15 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 
 			var file AdvisoryFile
 
-			if sha256 != "" || sha512 != "" || sign != "" {
+			switch {
-				file = HashedAdvisoryFile{self, sha256, sha512, sign}
+			case sha256 == "" && sha512 == "":
-			} else {
+				slog.Error("No hash listed on ROLIE feed", "file", self)
-				file = PlainAdvisoryFile(self)
+				return
+			case sign == "":
+				slog.Error("No signature listed on ROLIE feed", "file", self)
+				return
+			default:
+				file = PlainAdvisoryFile{self, sha256, sha512, sign}
 			}
 
 			files = append(files, file)

csaf/advisory.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -14,6 +14,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+
+	"github.com/gocsaf/csaf/v3/internal/misc"
 )
 
 // Acknowledgement reflects the 'acknowledgement' object in the list of acknowledgements.
@@ -383,7 +385,6 @@ type Relationship struct {
 	FullProductName           *FullProductName      `json:"full_product_name"`            // required
 	ProductReference          *ProductID            `json:"product_reference"`            // required
 	RelatesToProductReference *ProductID            `json:"relates_to_product_reference"` // required
-
 }
 
 // Relationships is a list of Relationship.
@@ -443,10 +444,11 @@ var csafFlagLabelPattern = alternativesUnmarshal(
 // machine readable flag. For example, this could be a machine readable justification
 // code why a product is not affected.
 type Flag struct {
-	Date       *string        `json:"date,omitempty"`
+	Date     *string        `json:"date,omitempty"`
-	GroupIDs   *ProductGroups `json:"group_ids,omitempty"`
+	GroupIDs *ProductGroups `json:"group_ids,omitempty"`
-	Label      *FlagLabel     `json:"label"` // required
+	Label    *FlagLabel     `json:"label"` // required
-	ProductIds *Products      `json:"product_ids,omitempty"`
+	//revive:disable-next-line:var-naming  until new major version w fix
+	ProductIds *Products `json:"product_ids,omitempty"`
 }
 
 // Flags is a list if Flag elements.
@@ -605,14 +607,16 @@ type RestartRequired struct {
 
 // Remediation specifies details on how to handle (and presumably, fix) a vulnerability.
 type Remediation struct {
-	Category        *RemediationCategory `json:"category"` // required
+	Category     *RemediationCategory `json:"category"` // required
-	Date            *string              `json:"date,omitempty"`
+	Date         *string              `json:"date,omitempty"`
-	Details         *string              `json:"details"` // required
+	Details      *string              `json:"details"` // required
-	Entitlements    []*string            `json:"entitlements,omitempty"`
+	Entitlements []*string            `json:"entitlements,omitempty"`
-	GroupIds        *ProductGroups       `json:"group_ids,omitempty"`
+	//revive:disable:var-naming until new major version w fix
-	ProductIds      *Products            `json:"product_ids,omitempty"`
+	GroupIds   *ProductGroups `json:"group_ids,omitempty"`
-	RestartRequired *RestartRequired     `json:"restart_required,omitempty"`
+	ProductIds *Products      `json:"product_ids,omitempty"`
-	URL             *string              `json:"url,omitempty"`
+	//revive:enable
+	RestartRequired *RestartRequired `json:"restart_required,omitempty"`
+	URL             *string          `json:"url,omitempty"`
 }
 
 // Remediations is a list of Remediation elements.
@@ -738,11 +742,13 @@ var csafThreatCategoryPattern = alternativesUnmarshal(
 
 // Threat contains information about a vulnerability that can change with time.
 type Threat struct {
-	Category   *ThreatCategory `json:"category"` // required
+	Category *ThreatCategory `json:"category"` // required
-	Date       *string         `json:"date,omitempty"`
+	Date     *string         `json:"date,omitempty"`
-	Details    *string         `json:"details"` // required
+	Details  *string         `json:"details"` // required
-	GroupIds   *ProductGroups  `json:"group_ids,omitempty"`
+	//revive:disable:var-naming until new major version w fix
-	ProductIds *Products       `json:"product_ids,omitempty"`
+	GroupIds   *ProductGroups `json:"group_ids,omitempty"`
+	ProductIds *Products      `json:"product_ids,omitempty"`
+	//revive:enable
 }
 
 // Threats is a list of Threat elements.
@@ -885,8 +891,8 @@ func (rs Revisions) Validate() error {
 
 // Validate validates an Engine.
 func (e *Engine) Validate() error {
-	if e.Version == nil {
+	if e.Name == nil {
-		return errors.New("'version' is missing")
+		return errors.New("'name' is missing")
 	}
 	return nil
 }
@@ -1391,7 +1397,7 @@ func LoadAdvisory(fname string) (*Advisory, error) {
 	}
 	defer f.Close()
 	var advisory Advisory
-	if err := json.NewDecoder(f).Decode(&advisory); err != nil {
+	if err := misc.StrictJSONParse(f, &advisory); err != nil {
 		return nil, err
 	}
 	if err := advisory.Validate(); err != nil {

csaf/advisory_test.go

@@ -0,0 +1,46 @@
+package csaf
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestLoadAdvisory(t *testing.T) {
+	type args struct {
+		jsonDir string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name:    "Valid documents",
+			args:    args{jsonDir: "csaf-documents/valid"},
+			wantErr: false,
+		},
+		{
+			name:    "Garbage trailing data",
+			args:    args{jsonDir: "csaf-documents/trailing-garbage-data"},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := filepath.Walk("../testdata/"+tt.args.jsonDir, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.Mode().IsRegular() && filepath.Ext(info.Name()) == ".json" {
+					if _, err := LoadAdvisory(path); (err != nil) != tt.wantErr {
+						t.Errorf("LoadAdvisory() error = %v, wantErr %v", err, tt.wantErr)
+					}
+				}
+				return nil
+			}); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

csaf/cvss20enums.go

@@ -1,10 +1,5 @@
-// This file is Free Software under the MIT License
+// SPDX-License-Identifier: BSD-3-Clause
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// SPDX-FileCopyrightText: 2017 FIRST.ORG, INC.
-//
-// SPDX-License-Identifier: MIT
-//
-// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
-// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
 //
 // THIS FILE IS MACHINE GENERATED. EDIT WITH CARE!
 

csaf/cvss3enums.go

@@ -1,10 +1,5 @@
-// This file is Free Software under the MIT License
+// SPDX-License-Identifier: BSD-3-Clause
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// SPDX-FileCopyrightText: 2017 FIRST.ORG, INC.
-//
-// SPDX-License-Identifier: MIT
-//
-// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
-// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
 //
 // THIS FILE IS MACHINE GENERATED. EDIT WITH CARE!
 

csaf/doc.go

@@ -1,12 +1,16 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
 
-// Package csaf contains the core data models used by the csaf distribution.
+// Package csaf contains the core data models used by the csaf distribution
+// tools.
+//
+// See https://github.com/gocsaf/csaf/tab=readme-ov-file#use-as-go-library
+// about hints and limits for its use as a library.
 package csaf
 
 //go:generate go run ./generate_cvss_enums.go -o cvss20enums.go -i ./schema/cvss-v2.0.json -p CVSS20

csaf/generate_cvss_enums.go

@@ -1,9 +1,9 @@
 //go:build ignore
 
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -12,23 +12,24 @@ package main
 
 import (
 	"bytes"
-	"encoding/json"
 	"flag"
+	"fmt"
 	"go/format"
 	"log"
 	"os"
-	"slices"
+	"regexp"
+	"sort"
 	"strings"
 	"text/template"
+
+	"github.com/gocsaf/csaf/v3/internal/misc"
 )
 
-const tmplText = `// This file is Free Software under the MIT License
+// We from Intevation consider the source code parts in the following
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// template file as too insignificant to be a piece of work that gains
-//
+// "copyrights" protection in the European Union. So the license(s)
-// SPDX-License-Identifier: MIT
+// of the output files are fully determined by the input file.
-//
+const tmplText = `// {{ $.License }}
-// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
-// Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
 //
 // THIS FILE IS MACHINE GENERATED. EDIT WITH CARE!
 
@@ -69,6 +70,7 @@ type definition struct {
 }
 
 type schema struct {
+	License     []string               `json:"license"`
 	Definitions map[string]*definition `json:"definitions"`
 }
 
@@ -97,7 +99,7 @@ func loadSchema(filename string) (*schema, error) {
 	}
 	defer f.Close()
 	var s schema
-	if err := json.NewDecoder(f).Decode(&s); err != nil {
+	if err := misc.StrictJSONParse(f, &s); err != nil {
 		return nil, err
 	}
 	return &s, nil
@@ -135,11 +137,24 @@ func main() {
 			defs = append(defs, k)
 		}
 	}
-	slices.Sort(defs)
+	sort.Strings(defs)
+
+	license := "determine license(s) from input file and replace this line"
+
+	pattern := regexp.MustCompile(`Copyright \(c\) (\d+), FIRST.ORG, INC.`)
+	for _, line := range s.License {
+		if m := pattern.FindStringSubmatch(line); m != nil {
+			license = fmt.Sprintf(
+				"SPDX-License-Identifier: BSD-3-Clause\n"+
+					"// SPDX-FileCopyrightText: %s FIRST.ORG, INC.", m[1])
+			break
+		}
+	}
 
 	var source bytes.Buffer
 
 	check(tmpl.Execute(&source, map[string]any{
+		"License":     license,
 		"Prefix":      *prefix,
 		"Definitions": s.Definitions,
 		"Keys":        defs,

csaf/models.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -17,7 +17,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // TLPLabel is the traffic light policy of the CSAF.
@@ -575,7 +576,6 @@ func (d *Distribution) Validate() error {
 // Validate checks if the provider metadata is valid.
 // Returns an error if the validation fails otherwise nil.
 func (pmd *ProviderMetadata) Validate() error {
-
 	switch {
 	case pmd.CanonicalURL == nil:
 		return errors.New("canonical_url is mandatory")
@@ -695,8 +695,7 @@ func (pmd *ProviderMetadata) WriteTo(w io.Writer) (int64, error) {
 func LoadProviderMetadata(r io.Reader) (*ProviderMetadata, error) {
 
 	var pmd ProviderMetadata
-	dec := json.NewDecoder(r)
+	if err := misc.StrictJSONParse(r, &pmd); err != nil {
-	if err := dec.Decode(&pmd); err != nil {
 		return nil, err
 	}
 

csaf/providermetaloader.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2023 Intevation GmbH <https://intevation.de>
@@ -11,13 +11,14 @@ package csaf
 import (
 	"bytes"
 	"crypto/sha256"
-	"encoding/json"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"strings"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // ProviderMetadataLoader helps load provider-metadata.json from
@@ -32,7 +33,7 @@ type ProviderMetadataLoader struct {
 type ProviderMetadataLoadMessageType int
 
 const (
-	//JSONDecodingFailed indicates problems with JSON decoding
+	// JSONDecodingFailed indicates problems with JSON decoding
 	JSONDecodingFailed ProviderMetadataLoadMessageType = iota
 	// SchemaValidationFailed indicates a general problem with schema validation.
 	SchemaValidationFailed
@@ -45,7 +46,7 @@ const (
 	// WellknownSecurityMismatch indicates that the PMDs found under wellknown and
 	// in the security do not match.
 	WellknownSecurityMismatch
-	// IgnoreProviderMetadata indicates that a extra PMD was ignored.
+	// IgnoreProviderMetadata indicates that an extra PMD was ignored.
 	IgnoreProviderMetadata
 )
 
@@ -108,8 +109,50 @@ func NewProviderMetadataLoader(client util.Client) *ProviderMetadataLoader {
 	}
 }
 
-// Load loads a provider metadata for a given path.
+// Enumerate lists all PMD files that can be found under the given domain.
-// If the domain starts with `https://` it only attemps to load
+// As specified in CSAF 2.0, it looks for PMDs using the well-known URL and
+// the security.txt, and if no PMDs have been found, it also checks the DNS-URL.
+func (pmdl *ProviderMetadataLoader) Enumerate(domain string) []*LoadedProviderMetadata {
+
+	// Our array of PMDs to be found
+	var resPMDs []*LoadedProviderMetadata
+
+	// Check direct path
+	if strings.HasPrefix(domain, "https://") {
+		return []*LoadedProviderMetadata{pmdl.loadFromURL(domain)}
+	}
+
+	// First try the well-known path.
+	wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
+
+	wellknownResult := pmdl.loadFromURL(wellknownURL)
+
+	// Validate the candidate and add to the result array
+	if wellknownResult.Valid() {
+		slog.Debug("Found well known provider-metadata.json")
+		resPMDs = append(resPMDs, wellknownResult)
+	}
+
+	// Next load the PMDs from security.txt
+	secResults := pmdl.loadFromSecurity(domain)
+	slog.Info("Found provider metadata results in security.txt", "num", len(secResults))
+
+	for _, result := range secResults {
+		if result.Valid() {
+			resPMDs = append(resPMDs, result)
+		}
+	}
+
+	// According to the spec, only if no PMDs have been found, the should DNS URL be used
+	if len(resPMDs) > 0 {
+		return resPMDs
+	}
+	dnsURL := "https://csaf.data.security." + domain
+	return []*LoadedProviderMetadata{pmdl.loadFromURL(dnsURL)}
+}
+
+// Load loads one valid provider metadata for a given path.
+// If the domain starts with `https://` it only attempts to load
 // the data from that URL.
 func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata {
 
@@ -129,24 +172,12 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata
 	// We have a candidate.
 	if wellknownResult.Valid() {
 		wellknownGood = wellknownResult
+	} else {
+		pmdl.messages.AppendUnique(wellknownResult.Messages)
 	}
 
 	// Next load the PMDs from security.txt
-	secURL := "https://" + domain + "/.well-known/security.txt"
+	secGoods := pmdl.loadFromSecurity(domain)
-	secResults := pmdl.loadFromSecurity(secURL)
-
-	// Filter out the results which are valid.
-	var secGoods []*LoadedProviderMetadata
-
-	for _, result := range secResults {
-		if len(result.Messages) > 0 {
-			// If there where validation issues append them
-			// to the overall report
-			pmdl.messages.AppendUnique(pmdl.messages)
-		} else {
-			secGoods = append(secGoods, result)
-		}
-	}
 
 	// Mention extra CSAF entries in security.txt.
 	ignoreExtras := func() {
@@ -177,78 +208,88 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata
 				}
 			}
 			// Take the good well-known.
-			wellknownGood.Messages.AppendUnique(pmdl.messages)
+			wellknownGood.Messages = pmdl.messages
 			return wellknownGood
 		}
 
 		// Don't have well-known. Take first good from security.txt.
 		ignoreExtras()
-		secGoods[0].Messages.AppendUnique(pmdl.messages)
+		secGoods[0].Messages = pmdl.messages
 		return secGoods[0]
 	}
 
 	// If we have a good well-known take it.
 	if wellknownGood != nil {
-		wellknownGood.Messages.AppendUnique(pmdl.messages)
+		wellknownGood.Messages = pmdl.messages
 		return wellknownGood
 	}
 
 	// Last resort: fall back to DNS.
 	dnsURL := "https://csaf.data.security." + domain
-	return pmdl.loadFromURL(dnsURL)
+	dnsURLResult := pmdl.loadFromURL(dnsURL)
+	pmdl.messages.AppendUnique(dnsURLResult.Messages) // keep order of messages consistent (i.e. last occurred message is last element)
+	dnsURLResult.Messages = pmdl.messages
+	return dnsURLResult
 }
 
-// loadFromSecurity loads the PMDs mentioned in the security.txt.
+// loadFromSecurity loads the PMDs mentioned in the security.txt. Only valid PMDs are returned.
-func (pmdl *ProviderMetadataLoader) loadFromSecurity(path string) []*LoadedProviderMetadata {
+func (pmdl *ProviderMetadataLoader) loadFromSecurity(domain string) []*LoadedProviderMetadata {
 
-	res, err := pmdl.client.Get(path)
+	// If .well-known fails try legacy location.
-	if err != nil {
+	for _, path := range []string{
-		pmdl.messages.Add(
+		"https://" + domain + "/.well-known/security.txt",
-			HTTPFailed,
+		"https://" + domain + "/security.txt",
-			fmt.Sprintf("Fetching %q failed: %v", path, err))
+	} {
-		return nil
+		res, err := pmdl.client.Get(path)
-	}
+		if err != nil {
-	if res.StatusCode != http.StatusOK {
+			pmdl.messages.Add(
-		pmdl.messages.Add(
+				HTTPFailed,
-			HTTPFailed,
+				fmt.Sprintf("Fetching %q failed: %v", path, err))
-			fmt.Sprintf("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode))
-		return nil
-	}
-
-	// Extract all potential URLs from CSAF.
-	urls, err := func() ([]string, error) {
-		defer res.Body.Close()
-		return ExtractProviderURL(res.Body, true)
-	}()
-
-	if err != nil {
-		pmdl.messages.Add(
-			HTTPFailed,
-			fmt.Sprintf("Loading %q failed: %v", path, err))
-		return nil
-	}
-
-	var loaded []*LoadedProviderMetadata
-
-	// Load the URLs
-nextURL:
-	for _, url := range urls {
-		lpmd := pmdl.loadFromURL(url)
-		// If loading failed note it down.
-		if !lpmd.Valid() {
-			pmdl.messages.AppendUnique(lpmd.Messages)
 			continue
 		}
-		// Check for duplicates
+		if res.StatusCode != http.StatusOK {
-		for _, l := range loaded {
+			pmdl.messages.Add(
-			if l == lpmd {
+				HTTPFailed,
-				continue nextURL
+				fmt.Sprintf("Fetching %q failed: %s (%d)", path, res.Status, res.StatusCode))
-			}
+			continue
 		}
-		loaded = append(loaded, lpmd)
-	}
 
-	return loaded
+		// Extract all potential URLs from CSAF.
+		urls, err := func() ([]string, error) {
+			defer res.Body.Close()
+			return ExtractProviderURL(res.Body, true)
+		}()
+
+		if err != nil {
+			pmdl.messages.Add(
+				HTTPFailed,
+				fmt.Sprintf("Loading %q failed: %v", path, err))
+			continue
+		}
+
+		var loaded []*LoadedProviderMetadata
+
+		// Load the URLs
+	nextURL:
+		for _, url := range urls {
+			lpmd := pmdl.loadFromURL(url)
+			// If loading failed note it down.
+			if !lpmd.Valid() {
+				pmdl.messages.AppendUnique(lpmd.Messages)
+				continue
+			}
+			// Check for duplicates
+			for _, l := range loaded {
+				if l == lpmd {
+					continue nextURL
+				}
+			}
+			loaded = append(loaded, lpmd)
+		}
+
+		return loaded
+	}
+	return nil
 }
 
 // loadFromURL loads a provider metadata from a given URL.
@@ -281,7 +322,7 @@ func (pmdl *ProviderMetadataLoader) loadFromURL(path string) *LoadedProviderMeta
 
 	var doc any
 
-	if err := json.NewDecoder(tee).Decode(&doc); err != nil {
+	if err := misc.StrictJSONParse(tee, &doc); err != nil {
 		result.Messages.Add(
 			JSONDecodingFailed,
 			fmt.Sprintf("JSON decoding failed: %v", err))
@@ -310,7 +351,7 @@ func (pmdl *ProviderMetadataLoader) loadFromURL(path string) *LoadedProviderMeta
 	case len(errors) > 0:
 		result.Messages = []ProviderMetadataLoadMessage{{
 			Type:    SchemaValidationFailed,
-			Message: fmt.Sprintf("%s: Validating against JSON schema failed: %v", path, err),
+			Message: fmt.Sprintf("%s: Validating against JSON schema failed", path),
 		}}
 		for _, msg := range errors {
 			result.Messages.Add(

csaf/remotevalidation.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"sync"
 
+	"github.com/gocsaf/csaf/v3/internal/misc"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -180,7 +181,6 @@ func prepareCache(config string) (cache, error) {
 			return create()
 		}
 		return nil
-
 	}); err != nil {
 		db.Close()
 		return nil, err
@@ -256,7 +256,7 @@ func deserialize(value []byte) (*RemoteValidationResult, error) {
 	}
 	defer r.Close()
 	var rvr RemoteValidationResult
-	if err := json.NewDecoder(r).Decode(&rvr); err != nil {
+	if err := misc.StrictJSONParse(r, &rvr); err != nil {
 		return nil, err
 	}
 	return &rvr, nil
@@ -323,7 +323,7 @@ func (v *remoteValidator) Validate(doc any) (*RemoteValidationResult, error) {
 			// no cache -> process directly.
 			in = resp.Body
 		}
-		return json.NewDecoder(in).Decode(&rvr)
+		return misc.StrictJSONParse(in, &rvr)
 	}(); err != nil {
 		return nil, err
 	}

csaf/rolie.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -14,7 +14,8 @@ import (
 	"sort"
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/internal/misc"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 // ROLIEServiceWorkspaceCollectionCategoriesCategory is a category in a ROLIE service collection.
@@ -54,7 +55,7 @@ type ROLIEServiceDocument struct {
 // LoadROLIEServiceDocument loads a ROLIE service document from a reader.
 func LoadROLIEServiceDocument(r io.Reader) (*ROLIEServiceDocument, error) {
 	var rsd ROLIEServiceDocument
-	if err := json.NewDecoder(r).Decode(&rsd); err != nil {
+	if err := misc.StrictJSONParse(r, &rsd); err != nil {
 		return nil, err
 	}
 	return &rsd, nil
@@ -122,7 +123,7 @@ func (rcd *ROLIECategoryDocument) Merge(categories ...string) bool {
 // LoadROLIECategoryDocument loads a ROLIE category document from a reader.
 func LoadROLIECategoryDocument(r io.Reader) (*ROLIECategoryDocument, error) {
 	var rcd ROLIECategoryDocument
-	if err := json.NewDecoder(r).Decode(&rcd); err != nil {
+	if err := misc.StrictJSONParse(r, &rcd); err != nil {
 		return nil, err
 	}
 	return &rcd, nil
@@ -168,14 +169,22 @@ type Format struct {
 
 // Entry for ROLIE.
 type Entry struct {
-	ID        string    `json:"id"`
+	Base        *string          `json:"base,omitempty"`
-	Titel     string    `json:"title"`
+	LanguageTag *string          `json:"lang,omitempty"`
-	Link      []Link    `json:"link"`
+	Author      *json.RawMessage `json:"author,omitempty"`
-	Published TimeStamp `json:"published"`
+	Category    []ROLIECategory  `json:"category,omitempty"`
-	Updated   TimeStamp `json:"updated"`
+	Content     Content          `json:"content"`
-	Summary   *Summary  `json:"summary,omitempty"`
+	Contributor *json.RawMessage `json:"contributor,omitempty"`
-	Content   Content   `json:"content"`
+	ID          string           `json:"id"`
-	Format    Format    `json:"format"`
+	Link        []Link           `json:"link"`
+	Published   TimeStamp        `json:"published"`
+	Rights      *json.RawMessage `json:"rights,omitempty"`
+	Source      *json.RawMessage `json:"source,omitempty"`
+	Summary     *Summary         `json:"summary,omitempty"`
+	Titel       string           `json:"title"`
+	Updated     TimeStamp        `json:"updated"`
+	Format      Format           `json:"format"`
+	Property    *json.RawMessage `json:"property,omitempty"`
 }
 
 // FeedData is the content of the ROLIE feed.
@@ -195,9 +204,8 @@ type ROLIEFeed struct {
 
 // LoadROLIEFeed loads a ROLIE feed from a reader.
 func LoadROLIEFeed(r io.Reader) (*ROLIEFeed, error) {
-	dec := json.NewDecoder(r)
 	var rf ROLIEFeed
-	if err := dec.Decode(&rf); err != nil {
+	if err := misc.StrictJSONParse(r, &rf); err != nil {
 		return nil, err
 	}
 	return &rf, nil

csaf/schema/aggregator_json_schema.json

@@ -175,7 +175,7 @@
         "type": "object",
         "required": [
           "metadata",
-          "mirror",
+          "mirrors",
           "update_interval"
         ],
         "properties": {

csaf/schema/cvss-v2.0.json.license

@@ -0,0 +1,2 @@
+SPDX-License-Identifier: BSD-3-Clause
+SPDX-FileCopyrightText: 2017 FIRST.ORG, INC.

csaf/schema/cvss-v3.0.json.license

@@ -0,0 +1,2 @@
+SPDX-License-Identifier: BSD-3-Clause
+SPDX-FileCopyrightText: 2017 FIRST.ORG, INC.

csaf/schema/cvss-v3.1.json.license

@@ -0,0 +1,2 @@
+SPDX-License-Identifier: BSD-3-Clause
+SPDX-FileCopyrightText: 2021 FIRST.ORG, INC.

csaf/summary.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -11,7 +11,7 @@ package csaf
 import (
 	"time"
 
-	"github.com/csaf-poc/csaf_distribution/v3/util"
+	"github.com/gocsaf/csaf/v3/util"
 )
 
 const (

csaf/util.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -36,3 +36,64 @@ func ExtractProviderURL(r io.Reader, all bool) ([]string, error) {
 	}
 	return urls, nil
 }
+
+// CollectProductIdentificationHelpers returns a slice of all ProductIdentificationHelper
+// for a given ProductID.
+func (pt *ProductTree) CollectProductIdentificationHelpers(id ProductID) []*ProductIdentificationHelper {
+	var helpers []*ProductIdentificationHelper
+	pt.FindProductIdentificationHelpers(
+		id, func(helper *ProductIdentificationHelper) {
+			helpers = append(helpers, helper)
+		})
+	return helpers
+}
+
+// FindProductIdentificationHelpers calls visit on all ProductIdentificationHelper
+// for a given ProductID by iterating over all full product names and branches
+// recursively available in the ProductTree.
+func (pt *ProductTree) FindProductIdentificationHelpers(
+	id ProductID,
+	visit func(*ProductIdentificationHelper),
+) {
+	// Iterate over all full product names
+	if fpns := pt.FullProductNames; fpns != nil {
+		for _, fpn := range *fpns {
+			if fpn != nil &&
+				fpn.ProductID != nil && *fpn.ProductID == id &&
+				fpn.ProductIdentificationHelper != nil {
+				visit(fpn.ProductIdentificationHelper)
+			}
+		}
+	}
+
+	// Iterate over branches recursively
+	var recBranch func(b *Branch)
+	recBranch = func(b *Branch) {
+		if b == nil {
+			return
+		}
+		if fpn := b.Product; fpn != nil &&
+			fpn.ProductID != nil && *fpn.ProductID == id &&
+			fpn.ProductIdentificationHelper != nil {
+			visit(fpn.ProductIdentificationHelper)
+		}
+		for _, c := range b.Branches {
+			recBranch(c)
+		}
+	}
+	for _, b := range pt.Branches {
+		recBranch(b)
+	}
+
+	// Iterate over relationships
+	if rels := pt.RelationShips; rels != nil {
+		for _, rel := range *rels {
+			if rel != nil {
+				if fpn := rel.FullProductName; fpn != nil && fpn.ProductID != nil &&
+					*fpn.ProductID == id && fpn.ProductIdentificationHelper != nil {
+					visit(fpn.ProductIdentificationHelper)
+				}
+			}
+		}
+	}
+}

csaf/util_test.go

@@ -0,0 +1,182 @@
+// This file is Free Software under the Apache-2.0 License
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
+
+package csaf
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestProductTree_FindProductIdentificationHelpers(t *testing.T) {
+	type fields struct {
+		Branches         Branches
+		FullProductNames *FullProductNames
+		RelationShips    *Relationships
+	}
+	type args struct {
+		id ProductID
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   []*ProductIdentificationHelper
+	}{
+		{
+			name: "empty product tree",
+			args: args{
+				id: "CSAFPID-0001",
+			},
+			want: nil,
+		},
+		{
+			name: "product tree with matching full product names",
+			fields: fields{
+				FullProductNames: &FullProductNames{{
+					ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+					ProductIdentificationHelper: &ProductIdentificationHelper{
+						CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+					},
+				}},
+			},
+			args: args{
+				id: "CSAFPID-0001",
+			},
+			want: []*ProductIdentificationHelper{{
+				CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+			}},
+		},
+		{
+			name: "product tree with no matching full product names",
+			fields: fields{
+				FullProductNames: &FullProductNames{{
+					ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+					ProductIdentificationHelper: &ProductIdentificationHelper{
+						CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+					},
+				}},
+			},
+			args: args{
+				id: "CSAFPID-0002",
+			},
+			want: nil,
+		},
+		{
+			name: "product tree with matching branches",
+			fields: fields{
+				Branches: Branches{{
+					Name: &[]string{"beta"}[0],
+					Product: &FullProductName{
+						ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+						ProductIdentificationHelper: &ProductIdentificationHelper{
+							CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+						},
+					},
+					Branches: Branches{{
+						Name: &[]string{"beta-2"}[0],
+						Product: &FullProductName{
+							ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+							ProductIdentificationHelper: &ProductIdentificationHelper{
+								CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta-2:*:*:*:*:*:*"}[0],
+							},
+						},
+					}},
+				}},
+			},
+			args: args{
+				id: "CSAFPID-0001",
+			},
+			want: []*ProductIdentificationHelper{{
+				CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+			}, {
+				CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta-2:*:*:*:*:*:*"}[0],
+			}},
+		},
+		{
+			name: "product tree with no matching branches",
+			fields: fields{
+				Branches: Branches{{
+					Name: &[]string{"beta"}[0],
+					Product: &FullProductName{
+						ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+						ProductIdentificationHelper: &ProductIdentificationHelper{
+							CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+						},
+					},
+					Branches: Branches{{
+						Name: &[]string{"beta-2"}[0],
+						Product: &FullProductName{
+							ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+							ProductIdentificationHelper: &ProductIdentificationHelper{
+								CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta-2:*:*:*:*:*:*"}[0],
+							},
+						},
+					}},
+				}},
+			},
+			args: args{
+				id: "CSAFPID-0002",
+			},
+			want: nil,
+		},
+		{
+			name: "product tree with matching relationships",
+			fields: fields{
+				RelationShips: &Relationships{{
+					FullProductName: &FullProductName{
+						ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+						ProductIdentificationHelper: &ProductIdentificationHelper{
+							CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+						},
+					},
+				}},
+			},
+			args: args{
+				id: "CSAFPID-0001",
+			},
+			want: []*ProductIdentificationHelper{{
+				CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+			}},
+		},
+		{
+			name: "product tree with no matching relationships",
+			fields: fields{
+				RelationShips: &Relationships{{
+					FullProductName: &FullProductName{
+						ProductID: &[]ProductID{"CSAFPID-0001"}[0],
+						ProductIdentificationHelper: &ProductIdentificationHelper{
+							CPE: &[]CPE{"cpe:2.3:a:microsoft:internet_explorer:1.0.0:beta:*:*:*:*:*:*"}[0],
+						},
+					},
+				}},
+			},
+			args: args{
+				id: "CSAFPID-0002",
+			},
+			want: nil,
+		},
+	}
+
+	t.Parallel()
+	for _, testToRun := range tests {
+		test := testToRun
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			pt := &ProductTree{
+				Branches:         test.fields.Branches,
+				FullProductNames: test.fields.FullProductNames,
+				RelationShips:    test.fields.RelationShips,
+			}
+			if got := pt.CollectProductIdentificationHelpers(test.args.id); !reflect.DeepEqual(got, test.want) {
+				tt.Errorf("ProductTree.FindProductIdentificationHelpers() = %v, want %v",
+					got, test.want)
+			}
+		})
+	}
+}

csaf/validation.go

@@ -1,7 +1,7 @@
-// This file is Free Software under the MIT License
+// This file is Free Software under the Apache-2.0 License
-// without warranty, see README.md and LICENSES/MIT.txt for details.
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 //
-// SPDX-License-Identifier: MIT
+// SPDX-License-Identifier: Apache-2.0
 //
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
@@ -10,13 +10,17 @@ package csaf
 
 import (
 	"bytes"
+	"crypto/tls"
 	_ "embed" // Used for embedding.
-	"io"
+	"errors"
+	"fmt"
+	"net/http"
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
-	"github.com/santhosh-tekuri/jsonschema/v5"
+	"github.com/santhosh-tekuri/jsonschema/v6"
 )
 
 //go:embed schema/csaf_json_schema.json
@@ -64,13 +68,28 @@ var (
 	compiledRolieSchema      = compiledSchema{url: rolieSchemaURL}
 )
 
-// loadURL loads the content of an URL from embedded data or
+type schemaLoader http.Client
-// falls back to the global loader function of the jsonschema package.
+
-func loadURL(s string) (io.ReadCloser, error) {
+func (l *schemaLoader) loadHTTPURL(url string) (any, error) {
-	loader := func(data []byte) (io.ReadCloser, error) {
+	client := (*http.Client)(l)
-		return io.NopCloser(bytes.NewReader(data)), nil
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
 	}
-	switch s {
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s returned status code %d", url, resp.StatusCode)
+	}
+
+	return jsonschema.UnmarshalJSON(resp.Body)
+}
+
+// Load loads the schema from the specified url.
+func (l *schemaLoader) Load(url string) (any, error) {
+	loader := func(data []byte) (any, error) {
+		return jsonschema.UnmarshalJSON(bytes.NewReader(data))
+	}
+	switch url {
 	case csafSchemaURL:
 		return loader(csafSchema)
 	case cvss20SchemaURL:
@@ -86,14 +105,27 @@ func loadURL(s string) (io.ReadCloser, error) {
 	case rolieSchemaURL:
 		return loader(rolieSchema)
 	default:
-		return jsonschema.LoadURL(s)
+		// Fallback to http loader
+		return l.loadHTTPURL(url)
 	}
 }
 
+func newSchemaLoader(insecure bool) *schemaLoader {
+	httpLoader := schemaLoader(http.Client{
+		Timeout: 15 * time.Second,
+	})
+	if insecure {
+		httpLoader.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+	}
+	return &httpLoader
+}
+
 func (cs *compiledSchema) compile() {
 	c := jsonschema.NewCompiler()
-	c.AssertFormat = true
+	c.AssertFormat()
-	c.LoadURL = loadURL
+	c.UseLoader(newSchemaLoader(false))
 	cs.compiled, cs.err = c.Compile(cs.url)
 }
 
@@ -109,7 +141,8 @@ func (cs *compiledSchema) validate(doc any) ([]string, error) {
 		return nil, nil
 	}
 
-	valErr, ok := err.(*jsonschema.ValidationError)
+	var valErr *jsonschema.ValidationError
+	ok := errors.As(err, &valErr)
 	if !ok {
 		return nil, err
 	}
@@ -133,21 +166,21 @@ func (cs *compiledSchema) validate(doc any) ([]string, error) {
 		if pi != pj {
 			return pi < pj
 		}
-		return errs[i].Error < errs[j].Error
+		return errs[i].Error.String() < errs[j].Error.String()
 	})
 
 	res := make([]string, 0, len(errs))
 
 	for i := range errs {
 		e := &errs[i]
-		if e.Error == "" {
+		if e.Error == nil {
 			continue
 		}
 		loc := e.InstanceLocation
 		if loc == "" {
 			loc = e.AbsoluteKeywordLocation
 		}
-		res = append(res, loc+": "+e.Error)
+		res = append(res, loc+": "+e.Error.String())
 	}
 
 	return res, nil

docs/Development.md

@@ -1,5 +1,10 @@
 # Development
 
+## Supported Go versions
+
+We support the latest version and the one before
+the latest version of Go (currently 1.24 and 1.25).
+
 ## Generated files
 
 Some source code files are machine generated. At the moment these are only

docs/csaf_aggregator.md

@@ -16,6 +16,7 @@ Help Options:
 ```
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/aggregator.toml
 ~/.csaf_aggregator.toml
@@ -25,6 +26,7 @@ csaf_aggregator.toml
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 
 Usage example for a single run, to test if the config is good:
+
 ```bash
 ./csaf_aggregator -c docs/examples/aggregator.toml
 ```
@@ -62,7 +64,6 @@ SHELL=/bin/bash
 30 0-23 * * * $HOME/bin/csaf_aggregator --config /etc/csaf_aggregator.toml --interim >> /var/log/csaf_aggregator/interim.log 2>&1
 ```
 
-
 #### serve via web server
 
 Serve the paths where the aggregator writes its `html/` output
@@ -78,7 +79,6 @@ a template. For the aggregator the difference is that you can leave out
 the cgi-bin part, potentially commend out the TLS client parts and
 adjust the `root` path accordingly.
 
-
 ### config options
 
 The config file is written in [TOML](https://toml.io/en/v1.0.0).
@@ -118,10 +118,12 @@ Next we have two TOML _tables_:
 aggregator            // basic infos for the aggregator object
 remote_validator      // config for optional remote validation checker
 ```
+
 [See the provider config](csaf_provider.md#provider-options) about
 how to configure `remote_validator`.
 
 At last there is the TOML _array of tables_:
+
 ```
 providers             // each entry to be mirrored or listed
 ```
@@ -148,6 +150,9 @@ header
 
 Where valid `name` and `domain` settings are required.
 
+If no user agent is specified with `header = "user-agent:custom-agent/1.0"`
+then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If you want an entry to be listed instead of mirrored
 in a `aggregator.category == "aggregator"` instance,
 set `category` to `lister` in the entry.
@@ -165,19 +170,20 @@ To offer an easy way of assorting CSAF documents by criteria like
 document category, languages or values of the branch category within
 the product tree, ROLIE category values can be configured in `categories`.
 This can either
-be done using an array of strings taken literally or, by prepending `"expr:"`. 
+be done using an array of strings taken literally or, by prepending `"expr:"`.
-The latter is evaluated as JSONPath and the result will be added into the 
+The latter is evaluated as JSONPath and the result will be added into the
 categories document. For a more detailed explanation and examples,
 [refer to the provider config](csaf_provider.md#provider-options).
 
-
 #### Example config file
+
 <!-- MARKDOWN-AUTO-DOCS:START (CODE:src=../docs/examples/aggregator.toml) -->
 <!-- The below code snippet is automatically added from ../docs/examples/aggregator.toml -->
+
 ```toml
 workers = 2
 folder = "/var/csaf_aggregator"
-lock_file = "/var/csaf_aggregator/run.lock"
+lock_file = "/var/lock/csaf_aggregator/lock"
 web = "/var/csaf_aggregator/html"
 domain = "https://localhost:9443"
 rate = 10.0
@@ -187,6 +193,7 @@ insecure = true
 #interim_years =
 #passphrase =
 #write_indices = false
+#time_range =
 
 # specification requires at least two providers (default),
 # to override for testing, enable:
@@ -208,6 +215,7 @@ insecure = true
   create_service_document = true
 #  rate = 1.5
 #  insecure = true
+#  time_range =
 
 [[providers]]
   name = "local-dev-provider2"
@@ -217,8 +225,8 @@ insecure = true
   write_indices = true
   client_cert = "./../devca1/testclient1.crt"
   client_key = "./../devca1/testclient1-key.pem"
-#  client_passphrase =
+#  client_passphrase = # Limited and experimental, see downloader doc.
-# header =
+#  header =
 
 [[providers]]
   name = "local-dev-provider3"
@@ -226,16 +234,22 @@ insecure = true
 #  rate = 1.8
 #  insecure = true
   write_indices = true
-  # If aggregator.category == "aggregator", set for an entry that should
+  # If aggregator.category == "aggreator", set for an entry that should
   # be listed in addition:
   category = "lister"
-#  ignore_pattern = [".*white.*", ".*red.*"]
+# ignore_pattern = [".*white.*", ".*red.*"]
 ```
-<!-- MARKDOWN-AUTO-DOCS:END -->
 
+<!-- MARKDOWN-AUTO-DOCS:END -->
 
 #### Publish others' advisories
 
 In case you want to provide CSAF advisories from others
 that only qualify as CSAF publishers, see
 [how to use the `csaf_aggregator` as "CSAF proxy provider"](proxy-provider-for-aggregator.md).
+
+Some providers may limit the rate of requests that may be sent to retrieve advisories.
+This may cause issues with the aggregator. 
+In this case, the --rate option can be used to adjust the requests per second
+sent by each worker of the aggregator to an acceptable rate.
+(The rate that is considered acceptable depends on the provider.)

docs/csaf_checker.md

@@ -30,9 +30,12 @@ Help Options:
 
 Will check all given _domains_, by trying each as a CSAF provider.
 
+If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and checking proceeds from there.
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/checker.toml
 ~/.csaf_checker.toml
@@ -41,6 +44,7 @@ csaf_checker.toml
 
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 Supported options in config files:
+
 ```
 output              = ""
 format              = "json"
@@ -58,9 +62,10 @@ validator_preset    = ["mandatory"]
 ```
 
 Usage example:
-` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
+`./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
 
 Each performed check has a return type of either 0,1 or 2:
+
 ```
 type 0: success
 type 1: warning
@@ -70,9 +75,15 @@ type 2: error
 The checker result is a success if no checks resulted in type 2, and a failure otherwise.
 
 The option `timerange` allows to only check advisories from a given time
-interval. It can only be given once.  See the
+interval. It can only be given once. See the
 [downloader documentation](csaf_downloader.md#timerange-option) for details.
 
+Some providers may limit the rate of requests that may be sent to retrieve advisories.
+This may cause the checker to be unable to retrieve all advisories. In this case,
+the --rate option can be used to adjust the requests per second
+sent by the checker to an acceptable rate.
+(The rate that is considered acceptable depends on the provider.)
+
 
 You can ignore certain advisories while checking by specifying a list
 of regular expressions[^1] to match their URLs by using the `ignorepattern`
@@ -80,6 +91,7 @@ option.
 E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain
 the sub strings **white** or **red**.
 In the config file this has to be noted as:
+
 ```
 ignorepattern = [".*white.*", ".*red.*"]
 ```
@@ -88,7 +100,7 @@ ignorepattern = [".*white.*", ".*red.*"]
 
 The `role` given in the `provider-metadata.json` is not
 yet considered to change the overall result,
-see https://github.com/csaf-poc/csaf_distribution/issues/221 .
+see <https://github.com/gocsaf/csaf/issues/221> .
 
 If a provider hosts one or more advisories with a TLP level of AMBER or RED, then these advisories must be access protected.
 To check these advisories, authorization can be given via custom headers or certificates.

docs/csaf_downloader.md

@@ -1,4 +1,5 @@
 ## csaf_downloader
+
 A tool to download CSAF documents from CSAF providers.
 
 ### Usage
@@ -21,6 +22,7 @@ Application Options:
   -f, --folder=FOLDER                            Download into a given subFOLDER
   -i, --ignore_pattern=PATTERN                   Do not download files if their URLs match any of the given PATTERNs
   -H, --header=                                  One or more extra HTTP header fields
+      --enumerate_pmd_only                       If this flag is set to true, the downloader will only enumerate valid provider metadata files, but not download documents
       --validator=URL                            URL to validate documents remotely
       --validator_cache=FILE                     FILE to cache remote validations
       --validator_preset=PRESETS                 One or more PRESETS to validate remotely (default: [mandatory])
@@ -29,9 +31,10 @@ Application Options:
       --forward_header=                          One or more extra HTTP header fields used by forwarding
       --forward_queue=LENGTH                     Maximal queue LENGTH before forwarder (default: 5)
       --forward_insecure                         Do not check TLS certificates from forward endpoint
-      --logfile=FILE                             FILE to log downloading to (default: downloader.log)
+      --log_file=FILE                            FILE to log downloading to (default: downloader.log)
-      --loglevel=LEVEL[debug|info|warn|error]    LEVEL of logging details (default: info)
+      --log_level=LEVEL[debug|info|warn|error]   LEVEL of logging details (default: info)
   -c, --config=TOML-FILE                         Path to config TOML file
+      --preferred_hash=HASH[sha256|sha512]       HASH to prefer
 
 Help Options:
   -h, --help                                     Show this help message
@@ -39,6 +42,8 @@ Help Options:
 
 Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider.
 
+If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and downloading procedes from there.
 
 Increasing the number of workers opens more connections to the web servers
@@ -46,7 +51,14 @@ to download more advisories at once. This may improve the overall speed of the d
 However, since this also increases the load on the servers, their administrators could
 have taken countermeasures to limit this.
 
+For example, some providers may limit the rate of requests that may be sent to retrieve advisories.
+This may cause the downloader to be unable to retrieve all advisories. 
+In this case, the --rate option can be used to adjust the requests per second
+sent by the downloader to an acceptable rate.
+(The rate that is considered acceptable depends on the provider.)
+
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/downloader.toml
 ~/.csaf_downloader.toml
@@ -56,13 +68,14 @@ csaf_downloader.toml
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 
 Supported options in config files:
+
 ```
 # directory         # not set by default
 insecure            = false
 # client_cert       # not set by default
 # client_key        # not set by default
 # client_passphrase # not set by default
-ignoresigcheck      = false
+ignore_sigcheck     = false
 # rate              # set to unlimited
 worker              = 2
 # time_range        # not set by default
@@ -90,14 +103,16 @@ option.
 E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain
 the sub strings **white** or **red**.
 In the config file this has to be noted as:
+
 ```
 ignorepattern = [".*white.*", ".*red.*"]
 ```
 
 #### Timerange option
 
-The `timerange` parameter enables downloading advisories which last changes falls
+The `time_range` parameter enables downloading advisories
-into a given intervall. There are three possible notations:
+which last changes falls into a given intervall.
+There are three possible notations:
 
 1. Relative. If the given string follows the rules of a
    [Go duration](https://pkg.go.dev/time@go1.20.6#ParseDuration),
@@ -106,16 +121,18 @@ into a given intervall. There are three possible notations:
    and 'y' for years are recognized. In these cases only integer
    values are accepted without any fractions.
    Some examples:
+
    - `"3h"` means downloading the advisories that have changed in the last three hours.
-   - `"30m"`  .. changed within the last thirty minutes.
+   - `"30m"` .. changed within the last thirty minutes.
    - `"3M2m"` .. changed within the last three months and two minutes.
-   - `"2y"`   .. changed within the last two years.
+   - `"2y"` .. changed within the last two years.
 
 2. Absolute. If the given string is an RFC 3339 date timestamp
    the time interval between this date and now is used.
    E.g. `"2006-01-02"` means that all files between 2006 January 2nd and now going to being
    downloaded.
    Accepted patterns are:
+
    - `"2006-01-02T15:04:05Z"`
    - `"2006-01-02T15:04:05+07:00"`
    - `"2006-01-02T15:04:05-07:00"`
@@ -134,6 +151,7 @@ into a given intervall. There are three possible notations:
 All interval boundaries are inclusive.
 
 #### Forwarding
+
 The downloader is able to forward downloaded advisories and their checksums,
 OpenPGP signatures and validation results to an HTTP endpoint.  
 The details of the implemented API are described [here](https://github.com/mfd2007/csaf_upload_interface).  
@@ -149,7 +167,7 @@ key protection mechanism based on RFC 1423, see
 Thus it considered experimental and most likely to be removed
 in a future release. Please only use this option, if you fully understand
 the security implications!
-Note that for fully automated processes, it usually not make sense
+Note that for fully automated processes, it usually does not make sense
 to protect the client certificate's private key with a passphrase.
 Because the passphrase has to be accessible to the process anyway to run
 unattented. In this situation the processing environment should be secured

docs/csaf_provider.md

@@ -4,7 +4,7 @@ The [setup docs](../README.md#setup-trusted-provider)
 explain how to wire this up with nginx and where the config file lives.
 
 When installed, two endpoints are offered,
-and you should use the [csaf_uploader](../docs/csaf_uploader)
+and you should use the [csaf_uploader](../docs/csaf_uploader.md)
 to access them:
 
 ### /api/create
@@ -58,7 +58,8 @@ The following example file documents all available configuration options:
 # The following shows an example of a manually set prefix:
 #canonical_url_prefix  = "https://localhost"
 
-# Require users to use a password and a valid Client Certificate for write access.
+# Require users to use both
+# (1) a password and (2) a valid Client Certificate for write access.
 #certificate_and_password = false
 
 # Allow the user to send the request without having to send a passphrase
@@ -100,22 +101,12 @@ The following example file documents all available configuration options:
 #tlps = ["csaf", "white", "amber", "green", "red"]
 
 # Make the provider create a ROLIE service document.
-#create_service_document = true
+#create_service_document = false
 
 # Make the provider create a ROLIE category document from a list of strings.
 # If a list item starts with `expr:`
 #   the rest of the string is used as a JsonPath expression
 #   to extract a string from the incoming advisories.
-#   If the result of the expression is a string this string
-#   is used. If the result is an array each element of
-#   this array is tested if it is a string or an array.
-#   If this test fails the expression fails. If the
-#   test succeeds the rules are applied recursively to
-#   collect all strings in the result.
-#   Suggested expressions are:
-#   - vendor, product family and product names:  "expr:$.product_tree..branches[?(@.category==\"vendor\" || @.category==\"product_family\" || @.category==\"product_name\")].name"
-#   - CVEs: "expr:$.vulnerabilities[*].cve"
-#   - CWEs: "expr:$.vulnerabilities[*].cwe.id"
 # Strings not starting with `expr:` are taken verbatim.
 # By default no category documents are created.
 # This example provides an overview over the syntax,
@@ -151,5 +142,5 @@ contact_details = "Example Company can be reached at contact_us@example.com, or
 
 There is an experimental upload interface which works with a web browser.
 It is disabled by default, as there are known issues, notably:
- * https://github.com/csaf-poc/csaf_distribution/issues/43
+ * https://github.com/gocsaf/csaf/issues/43
- * https://github.com/csaf-poc/csaf_distribution/issues/256
+ * https://github.com/gocsaf/csaf/issues/256

docs/csaf_uploader.md

@@ -43,6 +43,12 @@ E.g. uploading a csaf-document
 
 which asks to enter a password interactively.
 
+To upload an already signed document, use the `-x` option
+```bash
+# Note: The file CSAF-document-1.json.asc must exist
+./csaf_uploader -x -a upload -I -t white -u https://localhost/cgi-bin/csaf_provider.go  CSAF-document-1.json
+```
+
 By default csaf_uploader will try to load a config file
 from the following places:
 

docs/csaf_validator.md

@@ -2,6 +2,16 @@
 
 is a tool to validate local advisories files against the JSON Schema and an optional remote validator.
 
+### Exit codes
+
+If no fatal error occurs the program will exit with an exit code `n` with the following conditions:
+
+- `n == 0`: all valid
+- `(n & 1) > 0`: a general error occurred, all other flags are unset (see logs for more information)
+- `(n & 2) > 0`: schema validation failed
+- `(n & 4) > 0`: no remote validator configured
+- `(n & 8) > 0`: failure in remote validation
+
 ### Usage
 
 ```

docs/development-ca.md

@@ -55,7 +55,7 @@ signing_key
 encryption_key
 non_repudiation
 
-dns_name = "*.local"
+dns_name = "*.test"
 dns_name = "localhost"
 
 serial = 010

docs/examples/aggregator.toml

@@ -5,6 +5,7 @@ web = "/var/csaf_aggregator/html"
 domain = "https://localhost:9443"
 rate = 10.0
 insecure = true
+#verbose = false
 #openpgp_private_key =
 #openpgp_public_key =
 #interim_years =
@@ -51,7 +52,7 @@ insecure = true
 #  rate = 1.8
 #  insecure = true
   write_indices = true
-  # If aggregator.category == "aggreator", set for an entry that should
+  # If aggregator.category == "aggregator", set for an entry that should
   # be listed in addition:
   category = "lister"
 # ignore_pattern = [".*white.*", ".*red.*"]

docs/provider-setup.md

@@ -78,6 +78,9 @@ server {
 
         # directory listings
         autoindex on;
+
+        # allow others web applications to get the static information
+        add_header Access-Control-Allow-Origin "*";
     }
 
     # enable CGI
@@ -115,7 +118,7 @@ sudo chmod g+r,o-rwx /etc/csaf/config.toml
 
 Here is a minimal example configuration,
 which you need to customize for a production setup,
-see the [options of `csaf_provider`](https://github.com/csaf-poc/csaf_distribution/blob/main/docs/csaf_provider.md).
+see the [options of `csaf_provider`](https://github.com/gocsaf/csaf/blob/main/docs/csaf_provider.md).
 
 <!-- MARKDOWN-AUTO-DOCS:START (CODE:src=../docs/scripts/setupProviderForITest.sh&lines=94-101) -->
 <!-- The below code snippet is automatically added from ../docs/scripts/setupProviderForITest.sh -->
@@ -141,7 +144,7 @@ on a GNU/Linux operating system.
 
 Create the folders:
 ```(shell)
-curl https://192.168.56.102/cgi-bin/csaf_provider.go/create --cert-type p12 --cert {clientCertificat.p12}
+curl https://192.168.56.102/cgi-bin/csaf_provider.go/api/create --cert-type p12 --cert {clientCertificat.p12}
 ```
 Replace {clientCertificate.p12} with the client certificate file
 in pkcs12 format which includes the corresponding key as well.
@@ -155,7 +158,7 @@ Again replacing `{clientCert.crt}` and `{clientKey.pem}` accordingly.
 
 
 To let nginx resolves the DNS record `csaf.data.security.domain.tld` to fulfill the [Requirement 10](https://docs.oasis-open.org/csaf/csaf/v2.0/cs01/csaf-v2.0-cs01.html#7110-requirement-10-dns-path) configure a new server block (virtual host) in a separated file under `/etc/nginx/available-sites/{DNSNAME}` like following:
-<!-- MARKDOWN-AUTO-DOCS:START (CODE:src=../docs/scripts/DNSConfigForItest.sh&lines=18-35) -->
+<!-- MARKDOWN-AUTO-DOCS:START (CODE:src=../docs/scripts/DNSConfigForItest.sh&lines=18-37) -->
 <!-- The below code snippet is automatically added from ../docs/scripts/DNSConfigForItest.sh -->
 ```sh
     server {

docs/proxy-provider-for-aggregator.md

@@ -5,7 +5,9 @@ calls it a *CSAF publisher*.
 
 After manually downloading the advisories from such a publisher,
 the tools here can be used to offer the CSAF files for automated downloading
-as *CSAF aggregator*. (The construct is called *CSAF proxy provider*. See [Section 7.2.5](https://docs.oasis-open.org/csaf/csaf/v2.0/csaf-v2.0.html#725-role-csaf-aggregator) for more details.)
+as *CSAF aggregator*. (The construct is called *CSAF proxy provider*.
+See [Section 7.2.5](https://docs.oasis-open.org/csaf/csaf/v2.0/os/csaf-v2.0-os.html#725-role-csaf-aggregator)
+for more details.)
 
 There are three necessary steps, easiest is to use
 one single virtual maschine (or container) per internal provider.

docs/scripts/DNSConfigForItest.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 #
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -28,6 +28,8 @@ echo "
 
         location = / {
                 try_files /.well-known/csaf/provider-metadata.json =404;
+                # allow others web applications to get the static information
+                add_header Access-Control-Allow-Origin "*";
         }
 
         access_log /var/log/nginx/dns-domain_access.log;

docs/scripts/Readme.md

@@ -1,23 +1,24 @@
-Scripts for assisting the Integration tests. They are written on Ubuntu 20.04 TLS amd64.
+Scripts for assisting the Integration tests.
+They were written on Ubuntu 20.04 LTS amd64 and also tested with 24.04 LTS.
 
-- `prepareUbunutForITest.sh` installs the required packages for the csaf_distribution integration tests on a naked ubuntu 20.04 LTS amd64.
+- `prepareUbuntuInstanceForITests.sh` installs the required packages for the csaf integration tests on a naked Ubuntu LTS amd64.
 
 - `TLSConfigsForITest.sh` generates a root CA and webserver cert by running `createRootCAForITest.sh` and `createWebserverCertForITest.sh`
 and configures nginx for serving TLS connections.
 
 - `TLSClientConfigsForITest.sh` generates client certificates by calling `createCCForITest.sh` which uses the root certificate initialized before with `createRootCAForITest.sh`. It configures nginx to enable the authentication with client certificate. (This assumes that the same folder name is used to create the root certificate)
 
-- `setupProviderForITest.sh` builds the csaf_provider, writes the required nginx configurations and create the initial folders. IT calls `uploadToProvider.sh` to upload some csaf example files to the provider.
+- `setupProviderForITest.sh` builds the `csaf_provider`, writes the required nginx configurations and create the initial folders. IT calls `uploadToProvider.sh` to upload some csaf example files to the provider.
 
-As creating the folders needs to authenticate with the csaf_provider, the configurations of TLS server and Client certificate authentication should be set. So it is recommended to call the scripts in this order: `TLSConfigsForITest.sh`, `TLSClientConfigsForITest.sh`, `setupProviderForITest.sh`
+As creating the folders needs to authenticate with the `csaf_provider`, the configurations of TLS server and Client certificate authentication should be set. So it is recommended to call the scripts in this order: `TLSConfigsForITest.sh`, `TLSClientConfigsForITest.sh`, `setupProviderForITest.sh`
 
-Calling example (as root):
+Calling example (as user with sudo privileges):
 ``` bash
-    curl --fail -O https://raw.githubusercontent.com/csaf-poc/csaf_distribution/main/docs/scripts/prepareUbuntuInstanceForITests.sh
+    curl --fail -O https://raw.githubusercontent.com/gocsaf/csaf/main/docs/scripts/prepareUbuntuInstanceForITests.sh
-    bash prepareUbuntuInstanceForITests.sh
+    sudo bash prepareUbuntuInstanceForITests.sh
 
-    git clone https://github.com/csaf-poc/csaf_distribution.git # --branch <name>
+    git clone https://github.com/gocsaf/csaf.git # --branch <name>
-    pushd csaf_distribution/docs/scripts/
+    pushd csaf/docs/scripts/
 
     export FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)"
     source ./TLSConfigsForITest.sh

docs/scripts/TLSClientConfigsForITest.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -18,7 +18,7 @@ set -e
 
 NGINX_CONFIG_PATH=/etc/nginx/sites-available/default
 
-cd ~/csaf_distribution/docs/scripts/
+cd ~/csaf/docs/scripts/
 source ./createCCForITest.sh
 
 echo '

docs/scripts/TLSConfigsForITest.sh

@@ -1,7 +1,7 @@
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -17,7 +17,7 @@ set -e
 
 NGINX_CONFIG_PATH=/etc/nginx/sites-available/default
 
-cd ~/csaf_distribution/docs/scripts/
+cd ~/csaf/docs/scripts/
 ## Create Root CA
 ./createRootCAForITest.sh
 

docs/scripts/createCCForITest.sh

@@ -1,7 +1,7 @@
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>

docs/scripts/createRootCAForITest.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>

docs/scripts/createWebserverCertForITest.sh

@@ -1,7 +1,7 @@
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>

docs/scripts/downloadExamples.sh

@@ -2,10 +2,10 @@
 #
 # Desc: Tries getting csaf 2.0 examples from api.github. Do not run too often!
 #
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>

docs/scripts/prepareUbuntuInstanceForITests.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 set -e
 
-# This script prepares a naked Ubuntu 20.04 LTS amd64
+# This script prepares a naked Ubuntu LTS amd64
-# for the csaf_distribution integration tests
+# for the csaf integration tests
 # by installing the required packages.
 
 apt update
-apt install -y make bash curl gnupg sed tar git nginx fcgiwrap gnutls-bin
+apt install -y make bash curl gnupg sed tar git nginx fcgiwrap gnutls-bin zip
 
 # Install Go from binary distribution
 latest_go="$(curl https://go.dev/VERSION\?m=text| head -1).linux-amd64.tar.gz"
@@ -14,19 +14,17 @@ curl -O https://dl.google.com/go/$latest_go
 rm -rf /usr/local/go # be sure that we do not have an old installation
 tar -C /usr/local -xzf $latest_go
 
-# Install newer Node.js version from nodesource
+# Install a current Node.js version from nodesource
 # as needed for https://github.com/secvisogram/csaf-validator-service
 # Instructions from
 #  https://github.com/nodesource/distributions/blob/master/README.md#debmanual
 KEYRING=/usr/share/keyrings/nodesource.gpg
-curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > "$KEYRING"
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor > "$KEYRING"
 gpg --no-default-keyring --keyring "$KEYRING" --list-keys
 chmod a+r /usr/share/keyrings/nodesource.gpg
 
-VERSION=node_16.x
+NODE_MAJOR=20
-DISTRO="$(lsb_release -s -c)"
+echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
-echo "deb [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | tee /etc/apt/sources.list.d/nodesource.list
-echo "deb-src [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | tee -a /etc/apt/sources.list.d/nodesource.list
 
 apt-get update
 apt-get install -y nodejs

docs/scripts/setupProviderForITest.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 #
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -17,7 +17,7 @@ sudo chgrp -R www-data  /var/www
 sudo chmod -R g+ws  /var/www
 
 export NGINX_CONFIG_PATH=/etc/nginx/sites-available/default
-export DNS_NAME=csaf.data.security.localhost
+export DNS_NAME=csaf.data.security.test
 
 sudo cp /usr/share/doc/fcgiwrap/examples/nginx.conf /etc/nginx/fcgiwrap.conf
 
@@ -61,6 +61,9 @@ echo "
 
         # directory listings
         autoindex on;
+
+        # allow others web applications to get the static information
+        add_header Access-Control-Allow-Origin "*";
 " > locationConfig.txt
 sudo sed -i "/^\s*location \/ {/r locationConfig.txt" $NGINX_CONFIG_PATH # Insert config inside location{}
 ./DNSConfigForItest.sh

docs/scripts/setupValidationService.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 #
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -21,7 +21,7 @@ echo '
 remote_validator= { "url" = "http://localhost:8082", "presets" = ["mandatory"], "cache" = "/var/lib/csaf/validations.db" }
 ' | sudo tee --append /etc/csaf/config.toml
 
-npm install pm2 -g
+sudo npm install pm2 -g
 
 pushd ~
 git clone https://github.com/secvisogram/csaf-validator-service.git

docs/scripts/testAggregator.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
-# This file is Free Software under the MIT License
+# This file is Free Software under the Apache-2.0 License
-# without warranty, see README.md and LICENSES/MIT.txt for details.
+# without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
 #
-# SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: Apache-2.0
 #
 # SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 # Software-Engineering: 2022 Intevation GmbH <https://intevation.de>
@@ -29,6 +29,6 @@ popd
 echo
 echo '=== run aggregator'
 
-cd ~/csaf_distribution/
+cd ~/csaf/
 sudo cp docs/examples/aggregator.toml /etc/csaf
 sudo ./bin-linux-amd64/csaf_aggregator -c /etc/csaf/aggregator.toml