diff --git a/.github/workflows/generate-markdown.yml b/.github/workflows/generate-markdown.yml index de3559a..a59c944 100644 --- a/.github/workflows/generate-markdown.yml +++ b/.github/workflows/generate-markdown.yml @@ -1,6 +1,7 @@ name: generate-markdown on: + workflow_dispatch: push: branches: - "main" diff --git a/.github/workflows/itest.yml b/.github/workflows/itest.yml index 4f6c35c..a457eb1 100644 --- a/.github/workflows/itest.yml +++ b/.github/workflows/itest.yml @@ -20,8 +20,10 @@ jobs: cp -r $GITHUB_WORKSPACE ~ cd ~ cd csaf_distribution/docs/scripts/ - env FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)" ./TLSConfigsForITest.sh - env FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)" ./TLSClientConfigsForITest.sh + # keep in sync with docs/scripts/Readme.md + export FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)" + source ./TLSConfigsForITest.sh + ./TLSClientConfigsForITest.sh ./setupProviderForITest.sh ./testAggregator.sh shell: bash diff --git a/cmd/csaf_uploader/main.go b/cmd/csaf_uploader/main.go index 9599ed7..cc2f8ab 100644 --- a/cmd/csaf_uploader/main.go +++ b/cmd/csaf_uploader/main.go @@ -45,7 +45,7 @@ type options struct { ClientKey *string `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE.pem"` PasswordInteractive bool `short:"i" long:"password-interactive" description:"Enter password interactively" no-ini:"true"` - PassphraseInteractive bool `short:"I" long:"passphrase-interacive" description:"Enter passphrase interactively" no-ini:"true"` + PassphraseInteractive bool `short:"I" long:"passphrase-interactive" description:"Enter OpenPGP key passphrase interactively" no-ini:"true"` Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"` diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index 9182dcb..ab1c553 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -1,12 +1,121 @@ ## csaf_aggregator -Following options are supported: +### Usage -| Options | Description | -| --------------------- | -------------------------------------------------------------- | -| -c, --config=CFG-FILE | File name of the configuration file (default: aggregator.toml) | -| -i, --interim | Perform an interim scan | -| --version | Display version of the binary | +``` + csaf_aggregator [OPTIONS] -Usage example: -``` ./csaf_aggregator -c docs/examples/aggregator.toml ``` +Application Options: + -c, --config=CFG-FILE File name of the configuration file (default: + aggregator.toml) + --version Display version of the binary + -i, --interim Perform an interim scan + +Help Options: + -h, --help Show this help message +``` + +Usage example for a single run, to test if the config is good: +```bash +./csaf_aggregator -c docs/examples/aggregator.toml +``` + +Once the config is good, you can run the aggregator periodically +in two modes. For instance using `cron` on Ubuntu and after placing +the config file in `/etc/csaf_aggregator.toml`: + +```bash +mkdir /var/log/csaf_aggregator +mkdir ~www-data/bin +cp bin-linux-amd64/csaf_aggregator ~www-data/bin/ +chown www-data.www-data -R ~www-data/bin /var/log/csaf_aggregator + +# list current crontab +crontab -u www-data -l +# edit crontab (add lines like example below) +crontab -u www-data -e +``` + +Crontab example, running the full mode one a day and updating +interim advisories every 60 minutes: + +```crontab +SHELL=/bin/bash +# run full mode in the night at 04:00 +0 4 * * * $HOME/bin/csaf_aggregator --config /etc/csaf_aggregator.toml >> /var/log/csaf_aggregator/full.log 2>&1 +# run in interim mode once per hour at 30 minutes, e.g. 00:30, 01:30, ... +30 0-23 * * * $HOME/bin/csaf_aggregator --config /etc/csaf_aggregator.toml --interim >> /var/log/csaf_aggregator/interim.log 2>&1 +``` + + +### config options + +The following options can be used in the config file in TOML format: + +``` +workers // number of parallel workers to start (default 10) +folder // target folder on disc for writing the downloaded documents +web // directory to be served by the webserver +domain // base url where the contents will be reachable from outside +rate // overall downloading limit per worker +insecure // do not check validity of TLS certificates +aggregator // table with basic infos for the aggregator object +providers // array of tables, each entry to be mirrored or listed +key // OpenPGP key +openpgp_url // URL where the OpenPGP public key part can be found +passphrase // passphrase of the OpenPGP key +lock_file // path to lockfile, to stop other instances if one is not done +interim_years // limiting the years for which interim documents are searched +allow_single_provider // debugging option +``` + +Rates are specified as floats in HTTPS operations per second. +0 means no limit. + +`providers` is an array of tables, each allowing +``` +name +domain +rate +insecure +``` + +#### Example config file + + +```toml +workers = 2 +folder = "/var/csaf_aggregator" +lock_file = "/var/csaf_aggregator/run.lock" +web = "/var/csaf_aggregator/html" +domain = "https://localhost:9443" +rate = 10.0 +insecure = true + +[aggregator] + category = "aggregator" + name = "Example Development CSAF Aggregator" + contact_details = "some @ somewhere" + issuing_authority = "This service is provided as it is. It is gratis for everybody." + namespace = "https://testnamespace.example.org" + +[[providers]] + name = "local-dev-provider" + domain = "localhost" +# rate = 1.5 +# insecure = true + +[[providers]] + name = "local-dev-provider2" + domain = "localhost" +# rate = 1.2 +# insecure = true + +#key = +#passphrase = + +# specification requires at least two providers (default), +# to override for testing, enable: +# allow_single_provider = true +``` + diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index 09d0656..6036a55 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -1,16 +1,21 @@ ## csaf_checker -Following options are supported: +### Usage -| Options | Description | -| ------------------------------------------ | ---------------------------------------------- | -| -o, --output=REPORT-FILE | File name of the generated report | -| -f, --format=[json | html] | -| -t, --tlp=[csaf\|white\|green\|amber\|red] | Format of report (default: json) | -| --insecure | o not check TLS certificates from provider | -| --client-cert=CERT-FILE | TLS client certificate file (PEM encoded data) | -| --client-key=KEY-FILE | TLS client private key file (PEM encoded data) | -| --version | Display version of the binary | +``` + csaf_checker [OPTIONS] + +Application Options: + -o, --output=REPORT-FILE File name of the generated report + -f, --format=[json|html] Format of report (default: json) + --insecure Do not check TLS certificates from provider + --client-cert=CERT-FILE TLS client certificate file (PEM encoded data) + --client-key=KEY-FILE TLS client private key file (PEM encoded data) + --version Display version of the binary + +Help Options: + -h, --help Show this help message +``` Usage example: ` ./csaf_checker example.com -f html -o check-results.html` diff --git a/docs/csaf_provider.md b/docs/csaf_provider.md index 311b006..4e8ca10 100644 --- a/docs/csaf_provider.md +++ b/docs/csaf_provider.md @@ -1,5 +1,11 @@ +`csaf_provider` implements the CGI interface for webservers +and reads its configuration from a TOML file. +The [setup docs](../README.md#setup-trusted-provider) +explain how to wire this up with nginx and where the config file lives. + ## Provider options -Following options are supported: + +Following options are supported in the config file: - password: Authentication password for accessing the CSAF provider. - key: The private OpenPGP key. diff --git a/docs/csaf_uploader.md b/docs/csaf_uploader.md index 5f95892..b95d64a 100644 --- a/docs/csaf_uploader.md +++ b/docs/csaf_uploader.md @@ -1,48 +1,60 @@ ## csaf_uploader -Following options are supported: - -| Options | Description | -| ------------------------------------------ | ------------------------------------------------------------------------------------------ | -| -a, --action=[upload\|create] | Action to perform (default: upload) | -| -u, --url=URL | URL of the CSAF provider (default:https://localhost/cgi-bin/csaf_provider.go) | -| -t, --tlp=[csaf\|white\|green\|amber\|red] | TLP of the feed (default: csaf) | -| -x, --external-signed | CSAF files are signed externally. Assumes .asc files beside CSAF files | -| -k, --key=KEY-FILE | OpenPGP key to sign the CSAF files | -| -p, --password=PASSWORD | Authentication password for accessing the CSAF provider | -| -P, --passphrase=PASSPHRASE | Passphrase to unlock the OpenPGP key | -| -i, --password-interactive | Enter password interactively | -| -I, --passphrase-interacive | Enter passphrase interactively | -| -c, --config=INI-FILE | Path to config ini file | -| --insecure | Do not check TLS certificates from provider | -| --client-cert | TLS client certificate file (PEM encoded data) | -| --client-key | TLS client private key file (PEM encoded data) | -| --version | Display version of the binary | -| -h, --help | Show help | - -E.g. creating the initial directiories and files +### Usage ``` -./csaf_uploader -a create -u http://localhost/cgi-bin/csaf_provider.go + csaf_uploader [OPTIONS] + +Application Options: + -a, --action=[upload|create] Action to perform (default: upload) + -u, --url=URL URL of the CSAF provider (default: + https://localhost/cgi-bin/csaf_provider.go) + -t, --tlp=[csaf|white|green|amber|red] TLP of the feed (default: csaf) + -x, --external-signed CSAF files are signed externally. Assumes .asc files + beside CSAF files. + -s, --no-schema-check Do not check files against CSAF JSON schema locally. + -k, --key=KEY-FILE OpenPGP key to sign the CSAF files + -p, --password=PASSWORD Authentication password for accessing the CSAF provider + -P, --passphrase=PASSPHRASE Passphrase to unlock the OpenPGP key + --client-cert=CERT-FILE.crt TLS client certificate file (PEM encoded data) + --client-key=KEY-FILE.pem TLS client private key file (PEM encoded data) + -i, --password-interactive Enter password interactively + -I, --passphrase-interactive Enter passphrase interactively + --insecure Do not check TLS certificates from provider + -c, --config=INI-FILE Path to config ini file + --version Display version of the binary + +Help Options: + -h, --help Show this help message +``` +E.g. creating the initial directiories and files + +```bash +./csaf_uploader -a create -u https://localhost/cgi-bin/csaf_provider.go ``` E.g. uploading a csaf-document -``` -./csaf_uploader -a upload -I -t white -u http://localhost/cgi-bin/csaf_provider.go CSAF-document-1.json +```bash +./csaf_uploader -a upload -I -t white -u https://localhost/cgi-bin/csaf_provider.go CSAF-document-1.json ``` -which asks to enter password interactively. +which asks to enter a password interactively. -csaf_uploader can be started with a config file like following: +By default csaf_uploader will try to load a config file +from the following places: ``` -./csaf_provider -c conf.ini + "~/.config/csaf/uploader.ini", + "~/.csaf_uploader.ini", + "csaf_uploader.ini", ``` -config.ini : +The command line options can be written in the init file, except: +`password-interactive`, `passphrase-interactive` and `config`. +An example: ``` action=create -u=http://localhost/cgi-bin/csaf_provider.go +u=https://localhost/cgi-bin/csaf_provider.go ``` diff --git a/docs/development-ca.md b/docs/development-ca.md index 20f4b35..483732c 100644 --- a/docs/development-ca.md +++ b/docs/development-ca.md @@ -41,7 +41,7 @@ certtool --generate-self-signed --load-privkey rootca-key.pem --outfile rootca-c ```sh -cd ~/${FOLDERNAME} +pushd ~/${FOLDERNAME} certtool --generate-privkey --outfile testserver-key.pem @@ -66,12 +66,14 @@ certtool --generate-certificate --load-privkey testserver-key.pem --outfile test cat testserver.crt rootca-cert.pem >bundle.crt -SSL_CERTIFICATE=$( +export SSL_CERTIFICATE=$( echo "$PWD/bundle.crt" ) -SSL_CERTIFICATE_KEY=$( +export SSL_CERTIFICATE_KEY=$( echo "$PWD/testserver-key.pem" ) + +popd ``` diff --git a/docs/examples/aggregator.toml b/docs/examples/aggregator.toml index b91bbf2..35e36f1 100644 --- a/docs/examples/aggregator.toml +++ b/docs/examples/aggregator.toml @@ -1,9 +1,11 @@ workers = 2 -folder = "/var/csaf-aggregator" -web = "/var/csaf-aggregator/html" +folder = "/var/csaf_aggregator" +lock_file = "/var/csaf_aggregator/run.lock" +web = "/var/csaf_aggregator/html" domain = "https://localhost:9443" rate = 10.0 insecure = true + [aggregator] category = "aggregator" name = "Example Development CSAF Aggregator" @@ -26,8 +28,7 @@ insecure = true #key = #passphrase = -# for testing, the specifiation requires at least two +# specification requires at least two providers (default), +# to override for testing, enable: # allow_single_provider = true - - diff --git a/docs/install-server-certificate.md b/docs/install-server-certificate.md index 56b4da9..6f4dd05 100644 --- a/docs/install-server-certificate.md +++ b/docs/install-server-certificate.md @@ -50,13 +50,11 @@ finds your your private key and the certificate chain. ```sh - listen 443 ssl default_server; # ipv4 - listen [::]:443 ssl http2 default_server; # ipv6 - ssl_certificate '${SSL_CERTIFICATE}'; # e.g. ssl_certificate /etc/ssl/csaf/bundle.crt ssl_certificate_key '${SSL_CERTIFICATE_KEY}'; # e.g. ssl_certificate_key /etc/ssl/csaf/testserver-key.pem; ssl_protocols TLSv1.2 TLSv1.3; +' > ~/${FOLDERNAME}/TLSConfigs.txt ``` diff --git a/docs/provider-setup.md b/docs/provider-setup.md index 2a63561..1e1dac2 100644 --- a/docs/provider-setup.md +++ b/docs/provider-setup.md @@ -24,7 +24,7 @@ chmod -R g+w . Modify the content of `/etc/nginx/fcgiwrap.conf` like following: - + ```sh # Include this file on your nginx.conf to support debian cgi-bin scripts using @@ -91,7 +91,7 @@ Rename and place the `csaf_provider` binary file under `/usr/lib/cgi-bin/csaf_pr Create configuration file under `/usr/lib/csaf/config.toml`: - + ```sh # upload_signature = true @@ -118,6 +118,37 @@ Or using the uploader: Replace {password} with the password used for the authentication with csaf_provider. This needs to set the `password` option in `config.toml`. +To let nginx resolves the DNS record `csaf.data.security.domain.tld` to fulfill the [Requirement 10](https://docs.oasis-open.org/csaf/csaf/v2.0/cs01/csaf-v2.0-cs01.html#7110-requirement-10-dns-path) configure a new server block (virtual host) in a separated file under `/etc/nginx/available-sites/{DNSNAME}` like following: + + +```sh + server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + + ssl_certificate '${SSL_CERTIFICATE}'; # e.g. ssl_certificate /etc/ssl/csaf/bundle.crt + ssl_certificate_key '${SSL_CERTIFICATE_KEY}'; # e.g. ssl_certificate_key /etc/ssl/csaf/testserver-key.pem; + + root /var/www/html; + + server_name ${DNS_NAME}; # e.g. server_name csaf.data.security.domain.tld; + + location / { + try_files /.well-known/csaf/provider-metadata.json =404; + } + + access_log /var/log/nginx/dns-domain_access.log; + error_log /var/log/nginx/dns-domain_error.log; +} +``` + + +Then create a symbolic link to enable the new server block: +```shell +ln -s /etc/nginx/sites-available/{DNSNAME} /etc/nginx/sites-enabled/ +``` +Replace {DNSNAME} with a server block file name. + ## Provider options Provider has many config options described as following: diff --git a/docs/scripts/DNSConfigForItest.sh b/docs/scripts/DNSConfigForItest.sh new file mode 100755 index 0000000..01c99a8 --- /dev/null +++ b/docs/scripts/DNSConfigForItest.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# This file is Free Software under the MIT License +# without warranty, see README.md and LICENSES/MIT.txt for details. +# +# SPDX-License-Identifier: MIT +# +# SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +# Software-Engineering: 2022 Intevation GmbH + +# This script adds a new server block with the given DNS-Record and ajdust the "/etc/hosts" to +# set the DNS-Record for the localhost for testing. + +set -e + +sudo touch /etc/nginx/sites-available/DNSConfig +echo " + server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + + ssl_certificate '${SSL_CERTIFICATE}'; # e.g. ssl_certificate /etc/ssl/csaf/bundle.crt + ssl_certificate_key '${SSL_CERTIFICATE_KEY}'; # e.g. ssl_certificate_key /etc/ssl/csaf/testserver-key.pem; + + root /var/www/html; + + server_name ${DNS_NAME}; # e.g. server_name csaf.data.security.domain.tld; + + location / { + try_files /.well-known/csaf/provider-metadata.json =404; + } + + access_log /var/log/nginx/dns-domain_access.log; + error_log /var/log/nginx/dns-domain_error.log; +} +" | sudo tee -a /etc/nginx/sites-available/DNSConfig + +sudo ln -s /etc/nginx/sites-available/DNSConfig /etc/nginx/sites-enabled/ + +echo " + 127.0.0.1 $DNS_NAME +" | sudo tee -a /etc/hosts diff --git a/docs/scripts/Readme.md b/docs/scripts/Readme.md index d5aeccd..638fa88 100644 --- a/docs/scripts/Readme.md +++ b/docs/scripts/Readme.md @@ -19,7 +19,10 @@ Calling example (as root): git clone https://github.com/csaf-poc/csaf_distribution.git pushd csaf_distribution/docs/scripts/ - env FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)" ./TLSConfigsForITest.sh - env FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)" ./TLSClientConfigsForITest.sh + export FOLDERNAME=devca1 ORGANAME="CSAF Tools Development (internal)" + source ./TLSConfigsForITest.sh + set +e # for an interactive shell, reverse set -e done by previous line + ./TLSClientConfigsForITest.sh ./setupProviderForITest.sh + ./testAggregator.sh ``` diff --git a/docs/scripts/TLSClientConfigsForITest.sh b/docs/scripts/TLSClientConfigsForITest.sh index b070628..59e596a 100755 --- a/docs/scripts/TLSClientConfigsForITest.sh +++ b/docs/scripts/TLSClientConfigsForITest.sh @@ -38,7 +38,7 @@ echo ' return 404; } } -'> clientCertificateConfigs.txt +'> ~/${FOLDERNAME}/clientCertificateConfigs.txt sudo sed -i "/^server {/r ${HOME}/${FOLDERNAME}/clientCertificateConfigs.txt" $NGINX_CONFIG_PATH diff --git a/docs/scripts/TLSConfigsForITest.sh b/docs/scripts/TLSConfigsForITest.sh old mode 100755 new mode 100644 index 7bd8862..9b1f183 --- a/docs/scripts/TLSConfigsForITest.sh +++ b/docs/scripts/TLSConfigsForITest.sh @@ -1,5 +1,3 @@ -#!/usr/bin/env bash - # This file is Free Software under the MIT License # without warranty, see README.md and LICENSES/MIT.txt for details. # @@ -35,14 +33,14 @@ echo ' ssl_certificate_key '${SSL_CERTIFICATE_KEY}'; # e.g. ssl_certificate_key /etc/ssl/csaf/testserver-key.pem; ssl_protocols TLSv1.2 TLSv1.3; -' > TLSConfigs.txt +' > ~/${FOLDERNAME}/TLSConfigs.txt # a second listener port for testing setup where someone wants to tunnel access # to an unpriviledged port and still have the same access url echo ' listen 8443 ssl default_server; # ipv4 listen [::]:8443 ssl http2 default_server; # ipv6 -' > TLS8443Configs.txt +' > ~/${FOLDERNAME}/TLS8443Configs.txt sudo cp $NGINX_CONFIG_PATH $NGINX_CONFIG_PATH.org sudo sed -i "/^server {/r ${HOME}/${FOLDERNAME}/TLSConfigs.txt" $NGINX_CONFIG_PATH diff --git a/docs/scripts/createWebserverCertForITest.sh b/docs/scripts/createWebserverCertForITest.sh index 01e927c..2cfabd4 100644 --- a/docs/scripts/createWebserverCertForITest.sh +++ b/docs/scripts/createWebserverCertForITest.sh @@ -8,7 +8,7 @@ set -e -cd ~/${FOLDERNAME} +pushd ~/${FOLDERNAME} certtool --generate-privkey --outfile testserver-key.pem @@ -33,9 +33,11 @@ certtool --generate-certificate --load-privkey testserver-key.pem --outfile test cat testserver.crt rootca-cert.pem >bundle.crt -SSL_CERTIFICATE=$( +export SSL_CERTIFICATE=$( echo "$PWD/bundle.crt" ) -SSL_CERTIFICATE_KEY=$( +export SSL_CERTIFICATE_KEY=$( echo "$PWD/testserver-key.pem" ) + +popd diff --git a/docs/scripts/setupProviderForITest.sh b/docs/scripts/setupProviderForITest.sh index 15fbab6..76f1b9c 100755 --- a/docs/scripts/setupProviderForITest.sh +++ b/docs/scripts/setupProviderForITest.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +# # This file is Free Software under the MIT License # without warranty, see README.md and LICENSES/MIT.txt for details. # @@ -16,7 +16,8 @@ set -e sudo chgrp -R www-data /var/www sudo chmod -R g+w /var/www -NGINX_CONFIG_PATH=/etc/nginx/sites-available/default +export NGINX_CONFIG_PATH=/etc/nginx/sites-available/default +export DNS_NAME=csaf.data.security.domain.localhost sudo cp /usr/share/doc/fcgiwrap/examples/nginx.conf /etc/nginx/fcgiwrap.conf @@ -62,7 +63,7 @@ echo " autoindex on; " > locationConfig.txt sudo sed -i "/^\s*location \/ {/r locationConfig.txt" $NGINX_CONFIG_PATH # Insert config inside location{} - +./DNSConfigForItest.sh sudo systemctl reload nginx # assuming that we are in a checked out version in the docs/scripts directory @@ -94,3 +95,8 @@ popd # Upload files ./uploadToProvider.sh + +# Test resolving DNS record +curl https://$DNS_NAME --insecure + +./testChecker.sh diff --git a/docs/scripts/testChecker.sh b/docs/scripts/testChecker.sh new file mode 100755 index 0000000..12bd5c2 --- /dev/null +++ b/docs/scripts/testChecker.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# This file is Free Software under the MIT License +# without warranty, see README.md and LICENSES/MIT.txt for details. +# +# SPDX-License-Identifier: MIT +# +# SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +# Software-Engineering: 2022 Intevation GmbH + +set -e # to exit if a command in the script fails + +cd ~/csaf_distribution + +./bin-linux-amd64/csaf_checker -o show.html --insecure \ +--client-cert ~/devca1/testclient1.crt --client-key \ +~/devca1/testclient1-key.pem localhost -f html \ + +cat show.html