SystemSh0cker/gocsaf
1
0
Fork 0
mirror of https://github.com/gocsaf/csaf.git synced 2025-12-22 05:40:11 +01:00
Code Issues Releases Activity Actions

Add concurrent downloads to downloader. (#363)

Browse source 
* Add concurrent downloads to downloader.

* Moved to Go 1.20

* close files channel on producer side.

* Improve error handling

* New flag to ignore signature check results. Improve docs. Do not use number of CPUs to determine number of download workers.

* Set number of default workers in downloader to two.
This commit is contained in:
Sascha L. Teichmann 2023-05-02 10:10:12 +02:00 • committed by GitHub
parent 91479c9912
commit f32fba683d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 175 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

28
cmd/csaf_downloader/main.go
View file

@ -10,21 +10,27 @@
package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"github.com/csaf-poc/csaf_distribution/util"
"github.com/jessevdk/go-flags"
)
const defaultWorker = 2
type options struct {
Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"`
Version bool `long:"version" description:"Display version of the binary"`
Verbose bool `long:"verbose" short:"v" description:"Verbose output"`
Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"`
Directory *string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider"`
IgnoreSignatureCheck bool `long:"ignoresigcheck" description:"Ignore signature check results, just warn on mismatch"`
Version bool `long:"version" description:"Display version of the binary"`
Verbose bool `long:"verbose" short:"v" description:"Verbose output"`
Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)"`
Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM"`
ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields"`
@ -48,12 +54,20 @@ func run(opts *options, domains []string) error {
return err
}
defer d.close()
return d.run(domains)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx, stop := signal.NotifyContext(ctx, os.Interrupt)
defer stop()
return d.run(ctx, domains)
}
func main() {
opts := new(options)
opts := &options{
Worker: defaultWorker,
}
parser := flags.NewParser(opts, flags.Default)
parser.Usage = "[OPTIONS] domain..."