diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go index 9808542..b73286c 100644 --- a/cmd/csaf_aggregator/config.go +++ b/cmd/csaf_aggregator/config.go @@ -301,6 +301,11 @@ func (c *config) httpClient(p *provider) util.Client { Client: client, Header: c.ExtraHeader, } + default: + client = &util.HeaderClient{ + Client: client, + Header: http.Header{}, + } } if c.Verbose { diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 451a315..49e815c 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -435,11 +435,9 @@ func (p *processor) fullClient() util.Client { client := util.Client(&hClient) // Add extra headers. - if len(p.cfg.ExtraHeader) > 0 { - client = &util.HeaderClient{ - Client: client, - Header: p.cfg.ExtraHeader, - } + client = &util.HeaderClient{ + Client: client, + Header: p.cfg.ExtraHeader, } // Add optional URL logging. diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index badf060..9cef294 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -126,11 +126,9 @@ func (d *downloader) httpClient() util.Client { client := util.Client(&hClient) // Add extra headers. - if len(d.cfg.ExtraHeader) > 0 { - client = &util.HeaderClient{ - Client: client, - Header: d.cfg.ExtraHeader, - } + client = &util.HeaderClient{ + Client: client, + Header: d.cfg.ExtraHeader, } // Add optional URL logging. diff --git a/cmd/csaf_downloader/forwarder.go b/cmd/csaf_downloader/forwarder.go index 3b1435a..c3681eb 100644 --- a/cmd/csaf_downloader/forwarder.go +++ b/cmd/csaf_downloader/forwarder.go @@ -111,11 +111,9 @@ func (f *forwarder) httpClient() util.Client { client := util.Client(&hClient) // Add extra headers. - if len(f.cfg.ForwardHeader) > 0 { - client = &util.HeaderClient{ - Client: client, - Header: f.cfg.ForwardHeader, - } + client = &util.HeaderClient{ + Client: client, + Header: f.cfg.ForwardHeader, } // Add optional URL logging. diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index 36cbe7e..661871c 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -16,6 +16,7 @@ Help Options: ``` If no config file is explictly given the follwing places are searched for a config file: + ``` ~/.config/csaf/aggregator.toml ~/.csaf_aggregator.toml @@ -25,6 +26,7 @@ csaf_aggregator.toml with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems. Usage example for a single run, to test if the config is good: + ```bash ./csaf_aggregator -c docs/examples/aggregator.toml ``` @@ -62,7 +64,6 @@ SHELL=/bin/bash 30 0-23 * * * $HOME/bin/csaf_aggregator --config /etc/csaf_aggregator.toml --interim >> /var/log/csaf_aggregator/interim.log 2>&1 ``` - #### serve via web server Serve the paths where the aggregator writes its `html/` output @@ -78,7 +79,6 @@ a template. For the aggregator the difference is that you can leave out the cgi-bin part, potentially commend out the TLS client parts and adjust the `root` path accordingly. - ### config options The config file is written in [TOML](https://toml.io/en/v1.0.0). @@ -118,10 +118,12 @@ Next we have two TOML _tables_: aggregator // basic infos for the aggregator object remote_validator // config for optional remote validation checker ``` + [See the provider config](csaf_provider.md#provider-options) about how to configure `remote_validator`. At last there is the TOML _array of tables_: + ``` providers // each entry to be mirrored or listed ``` @@ -148,6 +150,9 @@ header Where valid `name` and `domain` settings are required. +If no user agent is specified with `header = "user-agent:custom-agent/1.0"` +then the default agent in the form of `csaf_distribution/VERSION` is sent. + If you want an entry to be listed instead of mirrored in a `aggregator.category == "aggregator"` instance, set `category` to `lister` in the entry. @@ -165,15 +170,16 @@ To offer an easy way of assorting CSAF documents by criteria like document category, languages or values of the branch category within the product tree, ROLIE category values can be configured in `categories`. This can either -be done using an array of strings taken literally or, by prepending `"expr:"`. -The latter is evaluated as JSONPath and the result will be added into the +be done using an array of strings taken literally or, by prepending `"expr:"`. +The latter is evaluated as JSONPath and the result will be added into the categories document. For a more detailed explanation and examples, [refer to the provider config](csaf_provider.md#provider-options). - #### Example config file + + ```toml workers = 2 folder = "/var/csaf_aggregator" @@ -233,8 +239,8 @@ insecure = true category = "lister" # ignore_pattern = [".*white.*", ".*red.*"] ``` - + #### Publish others' advisories diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index 58f77ca..0b223b6 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -30,9 +30,12 @@ Help Options: Will check all given _domains_, by trying each as a CSAF provider. +If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent. + If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and checking proceeds from there. If no config file is explictly given the follwing places are searched for a config file: + ``` ~/.config/csaf/checker.toml ~/.csaf_checker.toml @@ -41,6 +44,7 @@ csaf_checker.toml with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems. Supported options in config files: + ``` output = "" format = "json" @@ -58,9 +62,10 @@ validator_preset = ["mandatory"] ``` Usage example: -` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html` +`./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html` Each performed check has a return type of either 0,1 or 2: + ``` type 0: success type 1: warning @@ -70,16 +75,16 @@ type 2: error The checker result is a success if no checks resulted in type 2, and a failure otherwise. The option `timerange` allows to only check advisories from a given time -interval. It can only be given once. See the +interval. It can only be given once. See the [downloader documentation](csaf_downloader.md#timerange-option) for details. - You can ignore certain advisories while checking by specifying a list of regular expressions[^1] to match their URLs by using the `ignorepattern` option. E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain the sub strings **white** or **red**. In the config file this has to be noted as: + ``` ignorepattern = [".*white.*", ".*red.*"] ``` @@ -88,7 +93,7 @@ ignorepattern = [".*white.*", ".*red.*"] The `role` given in the `provider-metadata.json` is not yet considered to change the overall result, -see https://github.com/csaf-poc/csaf_distribution/issues/221 . +see . If a provider hosts one or more advisories with a TLP level of AMBER or RED, then these advisories must be access protected. To check these advisories, authorization can be given via custom headers or certificates. diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index fcf6634..003ae4a 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -1,4 +1,5 @@ ## csaf_downloader + A tool to download CSAF documents from CSAF providers. ### Usage @@ -39,6 +40,8 @@ Help Options: Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider. +If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent. + If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and downloading procedes from there. Increasing the number of workers opens more connections to the web servers @@ -47,6 +50,7 @@ However, since this also increases the load on the servers, their administrators have taken countermeasures to limit this. If no config file is explictly given the follwing places are searched for a config file: + ``` ~/.config/csaf/downloader.toml ~/.csaf_downloader.toml @@ -56,6 +60,7 @@ csaf_downloader.toml with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems. Supported options in config files: + ``` # directory # not set by default insecure = false @@ -90,6 +95,7 @@ option. E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain the sub strings **white** or **red**. In the config file this has to be noted as: + ``` ignorepattern = [".*white.*", ".*red.*"] ``` @@ -106,16 +112,18 @@ into a given intervall. There are three possible notations: and 'y' for years are recognized. In these cases only integer values are accepted without any fractions. Some examples: + - `"3h"` means downloading the advisories that have changed in the last three hours. - - `"30m"` .. changed within the last thirty minutes. + - `"30m"` .. changed within the last thirty minutes. - `"3M2m"` .. changed within the last three months and two minutes. - - `"2y"` .. changed within the last two years. + - `"2y"` .. changed within the last two years. 2. Absolute. If the given string is an RFC 3339 date timestamp the time interval between this date and now is used. E.g. `"2006-01-02"` means that all files between 2006 January 2nd and now going to being downloaded. Accepted patterns are: + - `"2006-01-02T15:04:05Z"` - `"2006-01-02T15:04:05+07:00"` - `"2006-01-02T15:04:05-07:00"` @@ -134,6 +142,7 @@ into a given intervall. There are three possible notations: All interval boundaries are inclusive. #### Forwarding + The downloader is able to forward downloaded advisories and their checksums, OpenPGP signatures and validation results to an HTTP endpoint. The details of the implemented API are described [here](https://github.com/mfd2007/csaf_upload_interface). diff --git a/util/client.go b/util/client.go index 5a11c7b..b4478ca 100644 --- a/util/client.go +++ b/util/client.go @@ -61,6 +61,11 @@ func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) { req.Header.Add(key, v) } } + + // Use default user agent if none is set + if userAgent := hc.Header.Get("User-Agent"); userAgent == "" { + req.Header.Add("User-Agent", "csaf_distribution/"+SemVersion) + } return hc.Client.Do(req) }