diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go index 1159977..84bc9fe 100644 --- a/cmd/csaf_aggregator/config.go +++ b/cmd/csaf_aggregator/config.go @@ -43,6 +43,7 @@ type provider struct { } type config struct { + Verbose bool `toml:"verbose"` // Workers is the number of concurrently executed workers for downloading. Workers int `toml:"workers"` Folder string `toml:"folder"` @@ -108,16 +109,25 @@ func (c *config) cryptoKey() (*crypto.Key, error) { func (c *config) httpClient(p *provider) util.Client { - client := http.Client{} + hClient := http.Client{} if p.Insecure != nil && *p.Insecure || c.Insecure != nil && *c.Insecure { - client.Transport = &http.Transport{ + hClient.Transport = &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } } + + var client util.Client + + if c.Verbose { + client = &util.LoggingClient{Client: &hClient} + } else { + client = &hClient + } + if p.Rate == nil && c.Rate == nil { - return &client + return client } var r float64 @@ -128,7 +138,7 @@ func (c *config) httpClient(p *provider) util.Client { r = *p.Rate } return &util.LimitingClient{ - Client: &client, + Client: client, Limiter: rate.NewLimiter(rate.Limit(r), 1), } } diff --git a/cmd/csaf_checker/main.go b/cmd/csaf_checker/main.go index 09d69c8..4b331ac 100644 --- a/cmd/csaf_checker/main.go +++ b/cmd/csaf_checker/main.go @@ -32,6 +32,7 @@ type options struct { ClientCert *string `long:"client-cert" description:"TLS client certificate file (PEM encoded data)" value-name:"CERT-FILE"` ClientKey *string `long:"client-key" description:"TLS client private key file (PEM encoded data)" value-name:"KEY-FILE"` Version bool `long:"version" description:"Display version of the binary"` + Verbose bool `long:"verbose" short:"v" description:"Verbose output"` Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second"` } diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 3ac3ff4..1a9c65e 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -270,9 +270,9 @@ func (p *processor) httpClient() util.Client { return p.client } - client := http.Client{} + hClient := http.Client{} - client.CheckRedirect = p.checkRedirect + hClient.CheckRedirect = p.checkRedirect var tlsConfig tls.Config if p.opts.Insecure { @@ -287,17 +287,25 @@ func (p *processor) httpClient() util.Client { tlsConfig.Certificates = []tls.Certificate{cert} } - client.Transport = &http.Transport{ + hClient.Transport = &http.Transport{ TLSClientConfig: &tlsConfig, } + var client util.Client + + if p.opts.Verbose { + client = &util.LoggingClient{Client: &hClient} + } else { + client = &hClient + } + if p.opts.Rate == nil { - p.client = &client - return &client + p.client = client + return client } p.client = &util.LimitingClient{ - Client: &client, + Client: client, Limiter: rate.NewLimiter(rate.Limit(*p.opts.Rate), 1), } diff --git a/csaf/util.go b/csaf/util.go index 2e2ae14..c221f99 100644 --- a/csaf/util.go +++ b/csaf/util.go @@ -136,7 +136,6 @@ func LoadProviderMetadataForDomain( // First try well-know path wellknownURL := "https://" + domain + "/.well-known/csaf/provider-metadata.json" - log.Printf("Trying: %s\n", wellknownURL) wellknownResult := LoadProviderMetadataFromURL(client, wellknownURL) if wellknownResult == nil { @@ -153,7 +152,6 @@ func LoadProviderMetadataForDomain( // Next load the PMDs from security.txt secURL := "https://" + domain + "/.well-known/security.txt" - log.Printf("Trying: %s\n", secURL) secResults := LoadProviderMetadatasFromSecurity(client, secURL) if secResults == nil { @@ -212,7 +210,6 @@ func LoadProviderMetadataForDomain( // Last resort fall back to DNS. dnsURL := "https://csaf.data.security." + domain - log.Printf("Trying: %s\n", dnsURL) dnsResult := LoadProviderMetadataFromURL(client, dnsURL) if dnsResult == nil { diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index ab1c553..a03008b 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -66,6 +66,7 @@ openpgp_url // URL where the OpenPGP public key part can be found passphrase // passphrase of the OpenPGP key lock_file // path to lockfile, to stop other instances if one is not done interim_years // limiting the years for which interim documents are searched +verbose // print more diagnostic output, e.g. https request allow_single_provider // debugging option ``` diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index 6036a55..9a342c8 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -12,10 +12,13 @@ Application Options: --client-cert=CERT-FILE TLS client certificate file (PEM encoded data) --client-key=KEY-FILE TLS client private key file (PEM encoded data) --version Display version of the binary + -v, --verbose Verbose output + -r, --rate= The average upper limit of https operations + per second Help Options: -h, --help Show this help message ``` Usage example: -` ./csaf_checker example.com -f html -o check-results.html` +` ./csaf_checker example.com -f html --rate=5.3 -o check-results.html` diff --git a/util/client.go b/util/client.go index 73edd5b..b0e5c59 100644 --- a/util/client.go +++ b/util/client.go @@ -11,6 +11,7 @@ package util import ( "context" "io" + "log" "net/http" "net/url" @@ -26,12 +27,47 @@ type Client interface { PostForm(url string, data url.Values) (*http.Response, error) } +// LoggingClient is a client that logs called URLs. +type LoggingClient struct { + Client +} + // LimitingClient is a Client implementing rate throttling. type LimitingClient struct { Client Limiter *rate.Limiter } +// Do implements the respective method of the Client interface. +func (lc *LoggingClient) Do(req *http.Request) (*http.Response, error) { + log.Printf("[DO]: %s\n", req.URL.String()) + return lc.Do(req) +} + +// Get implements the respective method of the Client interface. +func (lc *LoggingClient) Get(url string) (*http.Response, error) { + log.Printf("[GET]: %s\n", url) + return lc.Client.Get(url) +} + +// Head implements the respective method of the Client interface. +func (lc *LoggingClient) Head(url string) (*http.Response, error) { + log.Printf("[HEAD]: %s\n", url) + return lc.Head(url) +} + +// Post implements the respective method of the Client interface. +func (lc *LoggingClient) Post(url, contentType string, body io.Reader) (*http.Response, error) { + log.Printf("[POST]: %s\n", url) + return lc.Post(url, contentType, body) +} + +// PostForm implements the respective method of the Client interface. +func (lc *LoggingClient) PostForm(url string, data url.Values) (*http.Response, error) { + log.Printf("[POST FORM]: %s\n", url) + return lc.PostForm(url, data) +} + // Do implements the respective method of the Client interface. func (lc *LimitingClient) Do(req *http.Request) (*http.Response, error) { lc.Limiter.Wait(context.Background())