Merge pull request #553 from csaf-poc/user-agent

Use a default user agent

cmd/csaf_aggregator/config.go

@@ -301,6 +301,11 @@ func (c *config) httpClient(p *provider) util.Client {
 			Client: client,
 			Header: c.ExtraHeader,
 		}
+	default:
+		client = &util.HeaderClient{
+			Client: client,
+			Header: http.Header{},
+		}
 	}
 
 	if c.Verbose {

cmd/csaf_checker/processor.go

@@ -435,12 +435,10 @@ func (p *processor) fullClient() util.Client {
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(p.cfg.ExtraHeader) > 0 {
 	client = &util.HeaderClient{
 		Client: client,
 		Header: p.cfg.ExtraHeader,
 	}
-	}
 
 	// Add optional URL logging.
 	if p.cfg.Verbose {

cmd/csaf_downloader/downloader.go

@@ -126,12 +126,10 @@ func (d *downloader) httpClient() util.Client {
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(d.cfg.ExtraHeader) > 0 {
 	client = &util.HeaderClient{
 		Client: client,
 		Header: d.cfg.ExtraHeader,
 	}
-	}
 
 	// Add optional URL logging.
 	if d.cfg.verbose() {

cmd/csaf_downloader/forwarder.go

@@ -111,12 +111,10 @@ func (f *forwarder) httpClient() util.Client {
 	client := util.Client(&hClient)
 
 	// Add extra headers.
-	if len(f.cfg.ForwardHeader) > 0 {
 	client = &util.HeaderClient{
 		Client: client,
 		Header: f.cfg.ForwardHeader,
 	}
-	}
 
 	// Add optional URL logging.
 	if f.cfg.verbose() {

docs/csaf_aggregator.md

@@ -16,6 +16,7 @@ Help Options:
 ```
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/aggregator.toml
 ~/.csaf_aggregator.toml
@@ -25,6 +26,7 @@ csaf_aggregator.toml
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 
 Usage example for a single run, to test if the config is good:
+
 ```bash
 ./csaf_aggregator -c docs/examples/aggregator.toml
 ```
@@ -62,7 +64,6 @@ SHELL=/bin/bash
 30 0-23 * * * $HOME/bin/csaf_aggregator --config /etc/csaf_aggregator.toml --interim >> /var/log/csaf_aggregator/interim.log 2>&1
 ```
 
-
 #### serve via web server
 
 Serve the paths where the aggregator writes its `html/` output
@@ -78,7 +79,6 @@ a template. For the aggregator the difference is that you can leave out
 the cgi-bin part, potentially commend out the TLS client parts and
 adjust the `root` path accordingly.
 
-
 ### config options
 
 The config file is written in [TOML](https://toml.io/en/v1.0.0).
@@ -118,10 +118,12 @@ Next we have two TOML _tables_:
 aggregator            // basic infos for the aggregator object
 remote_validator      // config for optional remote validation checker
 ```
+
 [See the provider config](csaf_provider.md#provider-options) about
 how to configure `remote_validator`.
 
 At last there is the TOML _array of tables_:
+
 ```
 providers             // each entry to be mirrored or listed
 ```
@@ -148,6 +150,9 @@ header
 
 Where valid `name` and `domain` settings are required.
 
+If no user agent is specified with `header = "user-agent:custom-agent/1.0"`
+then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If you want an entry to be listed instead of mirrored
 in a `aggregator.category == "aggregator"` instance,
 set `category` to `lister` in the entry.
@@ -170,10 +175,11 @@ The latter is evaluated as JSONPath and the result will be added into the
 categories document. For a more detailed explanation and examples,
 [refer to the provider config](csaf_provider.md#provider-options).
 
-
 #### Example config file
+
 <!-- MARKDOWN-AUTO-DOCS:START (CODE:src=../docs/examples/aggregator.toml) -->
 <!-- The below code snippet is automatically added from ../docs/examples/aggregator.toml -->
+
 ```toml
 workers = 2
 folder = "/var/csaf_aggregator"
@@ -233,8 +239,8 @@ insecure = true
   category = "lister"
 # ignore_pattern = [".*white.*", ".*red.*"]
 ```
-<!-- MARKDOWN-AUTO-DOCS:END -->
 
+<!-- MARKDOWN-AUTO-DOCS:END -->
 
 #### Publish others' advisories
 

docs/csaf_checker.md

@@ -30,9 +30,12 @@ Help Options:
 
 Will check all given _domains_, by trying each as a CSAF provider.
 
+If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and checking proceeds from there.
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/checker.toml
 ~/.csaf_checker.toml
@@ -41,6 +44,7 @@ csaf_checker.toml
 
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 Supported options in config files:
+
 ```
 output              = ""
 format              = "json"
@@ -58,9 +62,10 @@ validator_preset    = ["mandatory"]
 ```
 
 Usage example:
-` ./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
+`./csaf_checker example.com -f html --rate=5.3 -H apikey:SECRET -o check-results.html`
 
 Each performed check has a return type of either 0,1 or 2:
+
 ```
 type 0: success
 type 1: warning
@@ -73,13 +78,13 @@ The option `timerange` allows to only check advisories from a given time
 interval. It can only be given once. See the
 [downloader documentation](csaf_downloader.md#timerange-option) for details.
 
-
 You can ignore certain advisories while checking by specifying a list
 of regular expressions[^1] to match their URLs by using the `ignorepattern`
 option.
 E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain
 the sub strings **white** or **red**.
 In the config file this has to be noted as:
+
 ```
 ignorepattern = [".*white.*", ".*red.*"]
 ```
@@ -88,7 +93,7 @@ ignorepattern = [".*white.*", ".*red.*"]
 
 The `role` given in the `provider-metadata.json` is not
 yet considered to change the overall result,
-see https://github.com/csaf-poc/csaf_distribution/issues/221 .
+see <https://github.com/csaf-poc/csaf_distribution/issues/221> .
 
 If a provider hosts one or more advisories with a TLP level of AMBER or RED, then these advisories must be access protected.
 To check these advisories, authorization can be given via custom headers or certificates.

docs/csaf_downloader.md

@@ -1,4 +1,5 @@
 ## csaf_downloader
+
 A tool to download CSAF documents from CSAF providers.
 
 ### Usage
@@ -39,6 +40,8 @@ Help Options:
 
 Will download all CSAF documents for the given _domains_, by trying each as a CSAF provider.
 
+If no user agent is specified with `--header=user-agent:custom-agent/1.0` then the default agent in the form of `csaf_distribution/VERSION` is sent.
+
 If a _domain_ starts with `https://` it is instead considered a direct URL to the `provider-metadata.json` and downloading procedes from there.
 
 Increasing the number of workers opens more connections to the web servers
@@ -47,6 +50,7 @@ However, since this also increases the load on the servers, their administrators
 have taken countermeasures to limit this.
 
 If no config file is explictly given the follwing places are searched for a config file:
+
 ```
 ~/.config/csaf/downloader.toml
 ~/.csaf_downloader.toml
@@ -56,6 +60,7 @@ csaf_downloader.toml
 with `~` expanding to `$HOME` on unixoid systems and `%HOMEPATH` on Windows systems.
 
 Supported options in config files:
+
 ```
 # directory         # not set by default
 insecure            = false
@@ -90,6 +95,7 @@ option.
 E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain
 the sub strings **white** or **red**.
 In the config file this has to be noted as:
+
 ```
 ignorepattern = [".*white.*", ".*red.*"]
 ```
@@ -106,6 +112,7 @@ into a given intervall. There are three possible notations:
    and 'y' for years are recognized. In these cases only integer
    values are accepted without any fractions.
    Some examples:
+
    - `"3h"` means downloading the advisories that have changed in the last three hours.
    - `"30m"` .. changed within the last thirty minutes.
    - `"3M2m"` .. changed within the last three months and two minutes.
@@ -116,6 +123,7 @@ into a given intervall. There are three possible notations:
    E.g. `"2006-01-02"` means that all files between 2006 January 2nd and now going to being
    downloaded.
    Accepted patterns are:
+
    - `"2006-01-02T15:04:05Z"`
    - `"2006-01-02T15:04:05+07:00"`
    - `"2006-01-02T15:04:05-07:00"`
@@ -134,6 +142,7 @@ into a given intervall. There are three possible notations:
 All interval boundaries are inclusive.
 
 #### Forwarding
+
 The downloader is able to forward downloaded advisories and their checksums,
 OpenPGP signatures and validation results to an HTTP endpoint.  
 The details of the implemented API are described [here](https://github.com/mfd2007/csaf_upload_interface).  

util/client.go

@@ -61,6 +61,11 @@ func (hc *HeaderClient) Do(req *http.Request) (*http.Response, error) {
 			req.Header.Add(key, v)
 		}
 	}
+
+	// Use default user agent if none is set
+	if userAgent := hc.Header.Get("User-Agent"); userAgent == "" {
+		req.Header.Add("User-Agent", "csaf_distribution/"+SemVersion)
+	}
 	return hc.Client.Do(req)
 }