Improve https get diagnostics, add verbose option

* Implement a logging client and activate it using verbose parameter or option
   in checker and aggregator.

Co-authored-by: Sascha L. Teichmann <sascha.teichmann@intevation.de>

cmd/csaf_aggregator/config.go

@@ -43,6 +43,7 @@ type provider struct {
 }
 
 type config struct {
+	Verbose bool `toml:"verbose"`
 	// Workers is the number of concurrently executed workers for downloading.
 	Workers int    `toml:"workers"`
 	Folder  string `toml:"folder"`
@@ -108,16 +109,25 @@ func (c *config) cryptoKey() (*crypto.Key, error) {
 
 func (c *config) httpClient(p *provider) util.Client {
 
-	client := http.Client{}
+	hClient := http.Client{}
 	if p.Insecure != nil && *p.Insecure || c.Insecure != nil && *c.Insecure {
-		client.Transport = &http.Transport{
+		hClient.Transport = &http.Transport{
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: true,
 			},
 		}
 	}
+
+	var client util.Client
+
+	if c.Verbose {
+		client = &util.LoggingClient{Client: &hClient}
+	} else {
+		client = &hClient
+	}
+
 	if p.Rate == nil && c.Rate == nil {
-		return &client
+		return client
 	}
 
 	var r float64
@@ -128,7 +138,7 @@ func (c *config) httpClient(p *provider) util.Client {
 		r = *p.Rate
 	}
 	return &util.LimitingClient{
-		Client:  &client,
+		Client:  client,
 		Limiter: rate.NewLimiter(rate.Limit(r), 1),
 	}
 }

cmd/csaf_checker/main.go

@@ -32,6 +32,7 @@ type options struct {
 	ClientCert *string  `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"`
 	ClientKey  *string  `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"`
 	Version    bool     `long:"version" description:"Display version of the binary"`
+	Verbose    bool     `long:"verbose" short:"v" description:"Verbose output"`
 	Rate       *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"`
 }
 

cmd/csaf_checker/processor.go

@@ -270,9 +270,9 @@ func (p *processor) httpClient() util.Client {
 		return p.client
 	}
 
-	client := http.Client{}
+	hClient := http.Client{}
 
-	client.CheckRedirect = p.checkRedirect
+	hClient.CheckRedirect = p.checkRedirect
 
 	var tlsConfig tls.Config
 	if p.opts.Insecure {
@@ -287,17 +287,25 @@ func (p *processor) httpClient() util.Client {
 		tlsConfig.Certificates = []tls.Certificate{cert}
 	}
 
-	client.Transport = &http.Transport{
+	hClient.Transport = &http.Transport{
 		TLSClientConfig: &tlsConfig,
 	}
 
+	var client util.Client
+
+	if p.opts.Verbose {
+		client = &util.LoggingClient{Client: &hClient}
+	} else {
+		client = &hClient
+	}
+
 	if p.opts.Rate == nil {
-		p.client = &client
+		p.client = client
-		return &client
+		return client
 	}
 
 	p.client = &util.LimitingClient{
-		Client:  &client,
+		Client:  client,
 		Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1),
 	}
 

csaf/util.go

@@ -136,7 +136,6 @@ func LoadProviderMetadataForDomain(
 
 	// First try well-know path
 	wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
-	log.Printf("Trying: %s\n", wellknownURL)
 	wellknownResult := LoadProviderMetadataFromURL(client, wellknownURL)
 
 	if wellknownResult == nil {
@@ -153,7 +152,6 @@ func LoadProviderMetadataForDomain(
 
 	// Next load the PMDs from security.txt
 	secURL := "https://" + domain + "/.well-known/security.txt"
-	log.Printf("Trying: %s\n", secURL)
 	secResults := LoadProviderMetadatasFromSecurity(client, secURL)
 
 	if secResults == nil {
@@ -212,7 +210,6 @@ func LoadProviderMetadataForDomain(
 	// Last resort fall back to DNS.
 
 	dnsURL := "https://csaf.data.security." + domain
-	log.Printf("Trying: %s\n", dnsURL)
 	dnsResult := LoadProviderMetadataFromURL(client, dnsURL)
 
 	if dnsResult == nil {

docs/csaf_aggregator.md

@@ -66,6 +66,7 @@ openpgp_url   // URL where the OpenPGP public key part can be found
 passphrase    // passphrase of the OpenPGP key
 lock_file     // path to lockfile, to stop other instances if one is not done
 interim_years // limiting the years for which interim documents are searched
+verbose       // print more diagnostic output, e.g. https request
 allow_single_provider // debugging option
 ```
 

docs/csaf_checker.md

@@ -12,10 +12,13 @@ Application Options:
       --client-cert=CERT-FILE    TLS client certificate file (PEM encoded data)
       --client-key=KEY-FILE      TLS client private key file (PEM encoded data)
       --version                  Display version of the binary
+  -v, --verbose                  Verbose output
+  -r, --rate=                    The average upper limit of https operations
+                                 per second
 
 Help Options:
   -h, --help                     Show this help message
 ```
 
 Usage example:
-` ./csaf_checker example.com -f html -o check-results.html`
+` ./csaf_checker example.com -f html --rate=5.3 -o check-results.html`

util/client.go

@@ -11,6 +11,7 @@ package util
 import (
 	"context"
 	"io"
+	"log"
 	"net/http"
 	"net/url"
 
@@ -26,12 +27,47 @@ type Client interface {
 	PostForm(url string, data url.Values) (*http.Response, error)
 }
 
+// LoggingClient is a client that logs called URLs.
+type LoggingClient struct {
+	Client
+}
+
 // LimitingClient is a Client implementing rate throttling.
 type LimitingClient struct {
 	Client
 	Limiter *rate.Limiter
 }
 
+// Do implements the respective method of the Client interface.
+func (lc *LoggingClient) Do(req *http.Request) (*http.Response, error) {
+	log.Printf("[DO]: %s\n", req.URL.String())
+	return lc.Do(req)
+}
+
+// Get implements the respective method of the Client interface.
+func (lc *LoggingClient) Get(url string) (*http.Response, error) {
+	log.Printf("[GET]: %s\n", url)
+	return lc.Client.Get(url)
+}
+
+// Head implements the respective method of the Client interface.
+func (lc *LoggingClient) Head(url string) (*http.Response, error) {
+	log.Printf("[HEAD]: %s\n", url)
+	return lc.Head(url)
+}
+
+// Post implements the respective method of the Client interface.
+func (lc *LoggingClient) Post(url, contentType string, body io.Reader) (*http.Response, error) {
+	log.Printf("[POST]: %s\n", url)
+	return lc.Post(url, contentType, body)
+}
+
+// PostForm implements the respective method of the Client interface.
+func (lc *LoggingClient) PostForm(url string, data url.Values) (*http.Response, error) {
+	log.Printf("[POST FORM]: %s\n", url)
+	return lc.PostForm(url, data)
+}
+
 // Do implements the respective method of the Client interface.
 func (lc *LimitingClient) Do(req *http.Request) (*http.Response, error) {
 	lc.Limiter.Wait(context.Background())