From 97ca513f9b512ff5204709fe7d6376ea94dae3f1 Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Mon, 17 Jan 2022 17:11:15 +0100 Subject: [PATCH 01/17] Add Makefile for building go components * Makefile with the following targets ** build for (linux system) ** build_win (windows system) ** build_tag (build from the last tag) ** clean for (removing the binaries) * Adjust README file --- Makefile | 39 +++++++++++++++++++++++++++++++++++++++ README.md | 10 ++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..914fc65 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +# Simple Make file to build csaf_distribution components + +SHELL=/bin/bash +BUILD = go build +buildMsg = "Building binaries..." + +.PHONY: build build_win build_tag clean + +all: + @echo choose a target from: build build_win build_tag clean + +# Build all the binaries and place them in the current directory level. +build: + @echo $(buildMsg) + @$(BUILD) -o ./ -v ./cmd/... + +# Build the binaries for windows and place them in the current directory level. +build_win: + @echo $(buildMsg) + @env GOOS=windows $(BUILD) -o ./ -v ./cmd/... + +# Build the binaries from the latest github tag. +TAG = $(shell git tag --sort=-version:refname | head -n 1) +build_tag: +ifeq ($(TAG),) + @echo "No Tag found" +else + @git checkout -q tags/${TAG}; + @echo $(buildMsg) + @$(BUILD) -o ./ -v ./cmd/...; + @env GOOS=windows $(BUILD) -o ./ -v ./cmd/... + @git checkout -q main +endif + +# Remove binary files +clean: + @rm -f csaf_checker csaf_provider csaf_uploader csaf_checker.exe csaf_provider.exe csaf_uploader.exe + + diff --git a/README.md b/README.md index f058e58..e1815ec 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,12 @@ - Clone the repository `git clone https://github.com/csaf-poc/csaf_distribution.git ` - Build Go components - ``` bash - cd csaf_distribution - go build -v ./cmd/... -``` + Makefile supplies the following builds: + - For Linux Systems :`make build` + - For Windows platform: `make build_win` + - Build from the last tag: `make build_tag` + +These places the binares in the current directory. - [Install](http://nginx.org/en/docs/install.html) **nginx** - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) From d37706e1e2262f7847318877cb229172935da4fd Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Tue, 18 Jan 2022 09:48:06 +0100 Subject: [PATCH 02/17] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1815ec..24e730c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ - For Windows platform: `make build_win` - Build from the last tag: `make build_tag` -These places the binares in the current directory. +These places the binaries in the current directory. - [Install](http://nginx.org/en/docs/install.html) **nginx** - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) From 84c3a108d068a8823d1526698e023dbcac596c1b Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Tue, 18 Jan 2022 16:16:33 +0100 Subject: [PATCH 03/17] Add some detail --- Makefile | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 914fc65..867d0a5 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ build: @echo $(buildMsg) @$(BUILD) -o ./ -v ./cmd/... -# Build the binaries for windows and place them in the current directory level. +# Build the binaries for windows (cross build) and place them in the current directory level. build_win: @echo $(buildMsg) @env GOOS=windows $(BUILD) -o ./ -v ./cmd/... diff --git a/README.md b/README.md index 24e730c..6928a5a 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ - Build Go components Makefile supplies the following builds: - - For Linux Systems :`make build` - - For Windows platform: `make build_win` + - For Linux System (default build):`make build` + - For Windows System (cross build): `make build_win` - Build from the last tag: `make build_tag` These places the binaries in the current directory. From ec6cabb9ac70bf5d5891a66138bff30dd83c9f99 Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Mon, 31 Jan 2022 13:09:47 +0100 Subject: [PATCH 04/17] Add one target to Makefile * "build_linux": building for GNU/linux * "build": Building for both linux and windows (cross build) * Place the generate binaries under "binaries/" directory * Improve echo messages --- Makefile | 29 +++++++++++++++++------------ README.md | 9 +++++---- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 867d0a5..02e3b20 100644 --- a/Makefile +++ b/Makefile @@ -2,22 +2,27 @@ SHELL=/bin/bash BUILD = go build -buildMsg = "Building binaries..." +MAKDIR = mkdir -p binaries .PHONY: build build_win build_tag clean all: - @echo choose a target from: build build_win build_tag clean + @echo choose a target from: build build_linux build_win build_tag clean -# Build all the binaries and place them in the current directory level. -build: - @echo $(buildMsg) - @$(BUILD) -o ./ -v ./cmd/... +# Build the binaries for GNU/linux and place them under binaries/ directory. +build_linux: + @$(MKDIR) + @echo "Bulding binaries for GNU/Linux ..." + @$(BUILD) -o ./binaries/ -v ./cmd/... -# Build the binaries for windows (cross build) and place them in the current directory level. +# Build the binaries for windows (cross build) and place them under binaries/ directory. build_win: - @echo $(buildMsg) - @env GOOS=windows $(BUILD) -o ./ -v ./cmd/... + @$(MKDIR) + @echo "Bulding binaries for windows (cross build) ..." + @env GOOS=windows $(BUILD) -o ./binaries/ -v ./cmd/... + +# Build the binaries for both GNU/linux and Windows and place them under binaries/ directory. +build: build_linux build_win # Build the binaries from the latest github tag. TAG = $(shell git tag --sort=-version:refname | head -n 1) @@ -27,13 +32,13 @@ ifeq ($(TAG),) else @git checkout -q tags/${TAG}; @echo $(buildMsg) - @$(BUILD) -o ./ -v ./cmd/...; + @$(BUILD) -o ./binaries/ -v ./cmd/...; @env GOOS=windows $(BUILD) -o ./ -v ./cmd/... @git checkout -q main endif -# Remove binary files +# Remove binaries directory clean: - @rm -f csaf_checker csaf_provider csaf_uploader csaf_checker.exe csaf_provider.exe csaf_uploader.exe + @rm -rf binaries/ diff --git a/README.md b/README.md index 6928a5a..ec23906 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,12 @@ - Build Go components Makefile supplies the following builds: - - For Linux System (default build):`make build` - - For Windows System (cross build): `make build_win` - - Build from the last tag: `make build_tag` + - Build For Linux System:`make build_linux` + - Build For Windows System (cross build): `make build_win` + - Build For both linux and windows: `make build` + - Build from the last github-tag: `make build_tag` -These places the binaries in the current directory. +These places the binaries under `binaries/` directory. - [Install](http://nginx.org/en/docs/install.html) **nginx** - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) From 0760901d6ece2b0ea723ede76889765f9dd721a8 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 31 Jan 2022 14:17:35 +0100 Subject: [PATCH 05/17] Fixed issue #37 --- csaf/validation.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/csaf/validation.go b/csaf/validation.go index f171234..c22906a 100644 --- a/csaf/validation.go +++ b/csaf/validation.go @@ -121,9 +121,15 @@ func (cs *compiledSchema) validate(doc interface{}) ([]string, error) { res := make([]string, 0, len(errs)) for i := range errs { - if e := &errs[i]; e.InstanceLocation != "" && e.Error != "" { - res = append(res, e.InstanceLocation+": "+e.Error) + e := &errs[i] + if e.Error == "" { + continue } + loc := e.InstanceLocation + if loc == "" { + loc = e.AbsoluteKeywordLocation + } + res = append(res, loc+": "+e.Error) } return res, nil From 9680a220be29565572ff79c3e9d05f46392c8b25 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 2 Feb 2022 11:20:51 +0100 Subject: [PATCH 06/17] Rename binaries folder to bin. --- Makefile | 8 ++++---- README.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 02e3b20..9669417 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL=/bin/bash BUILD = go build -MAKDIR = mkdir -p binaries +MKDIR = mkdir -p bin .PHONY: build build_win build_tag clean @@ -13,13 +13,13 @@ all: build_linux: @$(MKDIR) @echo "Bulding binaries for GNU/Linux ..." - @$(BUILD) -o ./binaries/ -v ./cmd/... + @$(BUILD) -o ./bin/ -v ./cmd/... # Build the binaries for windows (cross build) and place them under binaries/ directory. build_win: @$(MKDIR) @echo "Bulding binaries for windows (cross build) ..." - @env GOOS=windows $(BUILD) -o ./binaries/ -v ./cmd/... + @env GOARCH=amd64 GOOS=windows $(BUILD) -o ./bin/ -v ./cmd/... # Build the binaries for both GNU/linux and Windows and place them under binaries/ directory. build: build_linux build_win @@ -39,6 +39,6 @@ endif # Remove binaries directory clean: - @rm -rf binaries/ + @rm -rf bin/ diff --git a/README.md b/README.md index ec23906..1497e5f 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,13 @@ - Clone the repository `git clone https://github.com/csaf-poc/csaf_distribution.git ` - Build Go components - Makefile supplies the following builds: - - Build For Linux System:`make build_linux` + Makefile supplies the following targets: + - Build For GNU/Linux System: `make build_linux` - Build For Windows System (cross build): `make build_win` - Build For both linux and windows: `make build` - Build from the last github-tag: `make build_tag` -These places the binaries under `binaries/` directory. +These places the binaries under `bin/` directory. - [Install](http://nginx.org/en/docs/install.html) **nginx** - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) From 221d7269ec2aa67fbefc5240edc3f4fc1393af40 Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Thu, 3 Feb 2022 12:27:13 +0100 Subject: [PATCH 07/17] Replace "binaries" with "bin" in comments and one target --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9669417..58cc0c5 100644 --- a/Makefile +++ b/Makefile @@ -9,19 +9,19 @@ MKDIR = mkdir -p bin all: @echo choose a target from: build build_linux build_win build_tag clean -# Build the binaries for GNU/linux and place them under binaries/ directory. +# Build the binaries for GNU/linux and place them under bin/ directory. build_linux: @$(MKDIR) @echo "Bulding binaries for GNU/Linux ..." @$(BUILD) -o ./bin/ -v ./cmd/... -# Build the binaries for windows (cross build) and place them under binaries/ directory. +# Build the binaries for windows (cross build) and place them under bin/ directory. build_win: @$(MKDIR) @echo "Bulding binaries for windows (cross build) ..." @env GOARCH=amd64 GOOS=windows $(BUILD) -o ./bin/ -v ./cmd/... -# Build the binaries for both GNU/linux and Windows and place them under binaries/ directory. +# Build the binaries for both GNU/linux and Windows and place them under bin/ directory. build: build_linux build_win # Build the binaries from the latest github tag. @@ -32,12 +32,12 @@ ifeq ($(TAG),) else @git checkout -q tags/${TAG}; @echo $(buildMsg) - @$(BUILD) -o ./binaries/ -v ./cmd/...; + @$(BUILD) -o ./bin/ -v ./cmd/...; @env GOOS=windows $(BUILD) -o ./ -v ./cmd/... @git checkout -q main endif -# Remove binaries directory +# Remove bin/ directory clean: @rm -rf bin/ From b872f28acd63c7c6ede1518ad044d4df6b492251 Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Thu, 3 Feb 2022 15:18:40 +0100 Subject: [PATCH 08/17] Add some code documentation --- cmd/csaf_provider/actions.go | 6 ++++++ cmd/csaf_provider/config.go | 24 +++++++++++++++++++----- cmd/csaf_provider/create.go | 11 +++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index ccb85d8..60f117c 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -29,6 +29,8 @@ import ( const dateFormat = time.RFC3339 +// cleanFileName removes the "/" "\" charachters and replace the two or more +// occurences of "." with only one from the passed string. func cleanFileName(s string) string { s = strings.ReplaceAll(s, `/`, ``) s = strings.ReplaceAll(s, `\`, ``) @@ -37,6 +39,10 @@ func cleanFileName(s string) string { return s } +// loadCSAF loads the csaf file from the request, calls the "UploadLimter" function to +// set the upload limit size of the file and the "cleanFileName" to refine +// the filename. It returns the filename, file content in a buffer of bytes +// and an error. func (c *controller) loadCSAF(r *http.Request) (string, []byte, error) { file, handler, err := r.FormFile("csaf") if err != nil { diff --git a/cmd/csaf_provider/config.go b/cmd/csaf_provider/config.go index 65e1dce..920d5d6 100644 --- a/cmd/csaf_provider/config.go +++ b/cmd/csaf_provider/config.go @@ -21,14 +21,16 @@ import ( ) const ( + // The environment name, that contains the path to the config file. configEnv = "CSAF_CONFIG" - defaultConfigPath = "/usr/lib/casf/config.toml" - defaultFolder = "/var/www/" - defaultWeb = "/var/www/html" - defaultOpenPGPURL = "https://openpgp.circl.lu/pks/lookup?op=get&search=${FINGERPRINT}" - defaultUploadLimit = 50 * 1024 * 1024 + defaultConfigPath = "/usr/lib/casf/config.toml" // Default path to the config file. + defaultFolder = "/var/www/" // Default folder path. + defaultWeb = "/var/www/html" // Default web path. + defaultOpenPGPURL = "https://openpgp.circl.lu/pks/lookup?op=get&search=${FINGERPRINT}" // Default OpenPGP URL. + defaultUploadLimit = 50 * 1024 * 1024 // Default limit size of the uploaded file. ) +// configs contains the config values for the provider. type config struct { Password *string `toml:"password"` Key string `toml:"key"` @@ -56,6 +58,7 @@ const ( tlpRed tlp = "red" ) +// valid returns true if the checked tlp matches one of the defined tlps. func (t tlp) valid() bool { switch t { case tlpCSAF, tlpWhite, tlpGreen, tlpAmber, tlpRed: @@ -73,6 +76,8 @@ func (t *tlp) UnmarshalText(text []byte) error { return fmt.Errorf("invalid config TLP value: %v", string(text)) } +// uploadLimiter returns a reader that reads from a given r reader but stops +// with EOF after the defined bytes in the "UploadLimit" config option. func (cfg *config) uploadLimiter(r io.Reader) io.Reader { // Zero or less means no upload limit. if cfg.UploadLimit == nil || *cfg.UploadLimit < 1 { @@ -100,6 +105,8 @@ func (cfg *config) modelTLPs() []csaf.TLPLabel { return tlps } +// loadCryptoKey loads the armored data into the key stored in the file specified by the +// "key" config value and return it with nil, otherwise an error. func (cfg *config) loadCryptoKey() (*crypto.Key, error) { f, err := os.Open(cfg.Key) if err != nil { @@ -109,11 +116,18 @@ func (cfg *config) loadCryptoKey() (*crypto.Key, error) { return crypto.NewKeyFromArmoredReader(f) } +// checkPassword compares the given hashed password with the plaintext in the "password" config value. +// It returns true if these matches or if the "password" config value is not set, otherwise false. func (cfg *config) checkPassword(hash string) bool { return cfg.Password == nil || bcrypt.CompareHashAndPassword([]byte(hash), []byte(*cfg.Password)) == nil } +// loadConfig extracts the config values from the config file. The path to the +// file is taken either from environment variable "CSAF_CONFIG" or from the +// defined default path in "defaultConfigPath". +// Default values are set in case some are missing in the file. +// It returns these values in a struct and nil if there is no error. func loadConfig() (*config, error) { path := os.Getenv(configEnv) if path == "" { diff --git a/cmd/csaf_provider/create.go b/cmd/csaf_provider/create.go index 7507dfd..b4bf281 100644 --- a/cmd/csaf_provider/create.go +++ b/cmd/csaf_provider/create.go @@ -18,6 +18,8 @@ import ( "github.com/csaf-poc/csaf_distribution/util" ) +// ensureFolders initializes the paths and call functions to create +// the directories and files. func ensureFolders(c *config) error { wellknown := filepath.Join(c.Web, ".well-known") @@ -38,6 +40,8 @@ func ensureFolders(c *config) error { return createSecurity(c, wellknown) } +// createWellknown creates ".well-known" directory if not exist and returns nil. +// An error is returned if the it is not a directory. func createWellknown(wellknown string) error { st, err := os.Stat(wellknown) if err != nil { @@ -52,6 +56,10 @@ func createWellknown(wellknown string) error { return nil } +// createFeedFolders creates the feed folders according to the tlp values +// in the "tlps" config option if they do not already exist. +// No creation for the "csaf" option will be done. +// It creates also symbolic links to feed folders. func createFeedFolders(c *config, wellknown string) error { for _, t := range c.TLPs { if t == tlpCSAF { @@ -75,6 +83,8 @@ func createFeedFolders(c *config, wellknown string) error { return nil } +// createSecurity creats the "security.txt" file if does not exist +// and writes the CSAF field inside the file. func createSecurity(c *config, wellknown string) error { security := filepath.Join(wellknown, "security.txt") if _, err := os.Stat(security); err != nil { @@ -93,6 +103,7 @@ func createSecurity(c *config, wellknown string) error { return nil } +// createProviderMetadata creates the provider-metadata.json file if does not exist. func createProviderMetadata(c *config, wellknownCSAF string) error { path := filepath.Join(wellknownCSAF, "provider-metadata.json") _, err := os.Stat(path) From e8cbab2c2963745dc850f3c72944cc66175686b4 Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Mon, 7 Feb 2022 13:00:28 +0100 Subject: [PATCH 09/17] Add some documentation (controller) --- cmd/csaf_provider/actions.go | 2 ++ cmd/csaf_provider/controller.go | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index 60f117c..a4fc0a5 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -129,6 +129,8 @@ func (c *controller) tlpParam(r *http.Request) (tlp, error) { return "", fmt.Errorf("unsupported TLP type '%s'", t) } +// creates calls the "ensureFolders" functions to create the directories and files. +// It returns a struct by success, otherwise an error. func (c *controller) create(*http.Request) (interface{}, error) { if err := ensureFolders(c.cfg); err != nil { return nil, err diff --git a/cmd/csaf_provider/controller.go b/cmd/csaf_provider/controller.go index 5ea05f4..a5636be 100644 --- a/cmd/csaf_provider/controller.go +++ b/cmd/csaf_provider/controller.go @@ -37,11 +37,14 @@ func asMultiError(err error) multiError { return multiError([]string{err.Error()}) } +// controller contains the config values and the html templates. type controller struct { cfg *config tmpl *template.Template } +// newController assigns the given configs to a controller variable and parses the html template +// if the config value "NoWebUI" is true. It returns the controller variable and nil, otherwise error. func newController(cfg *config) (*controller, error) { c := controller{cfg: cfg} @@ -56,6 +59,8 @@ func newController(cfg *config) (*controller, error) { return &c, nil } +// bind binds the paths with the corresponding http.handler and wraps it with the respective middleware, +// according to the "NoWebUI" config value. func (c *controller) bind(pim *pathInfoMux) { if !c.cfg.NoWebUI { pim.handleFunc("/", c.auth(c.index)) @@ -66,6 +71,9 @@ func (c *controller) bind(pim *pathInfoMux) { pim.handleFunc("/api/create", c.auth(api(c.create))) } +// auth wraps the given http.HandlerFunc and returns an new one after authenticating the +// password contained in the header "X-CSAF-PROVIDER-AUTH" with the "password" config value +// if set, otherwise returns the given http.HandlerFunc. func (c *controller) auth( fn func(http.ResponseWriter, *http.Request), ) func(http.ResponseWriter, *http.Request) { @@ -83,6 +91,9 @@ func (c *controller) auth( } } +// render sets the headers for the response. It applies the given template "tmpl" to +// the given object "arg" and writes the output to http.ResponseWriter. +// It logs a warning in case of error. func (c *controller) render(rw http.ResponseWriter, tmpl string, arg interface{}) { rw.Header().Set("Content-type", "text/html; charset=utf-8") rw.Header().Set("X-Content-Type-Options", "nosniff") @@ -91,17 +102,24 @@ func (c *controller) render(rw http.ResponseWriter, tmpl string, arg interface{} } } +// failed constructs the error messages by calling "asMultiError" and calls "render" +// function to render the passed template and error object. func (c *controller) failed(rw http.ResponseWriter, tmpl string, err error) { result := map[string]interface{}{"Error": asMultiError(err)} c.render(rw, tmpl, result) } +// index calls the "render" function and passes the "index.html" and c.cfg to it. func (c *controller) index(rw http.ResponseWriter, r *http.Request) { c.render(rw, "index.html", map[string]interface{}{ "Config": c.cfg, }) } +// web executes the given function "fn", calls the "render" function and passes +// the result content from "fn", the given template and the http.ResponseWriter to it +// in case of no error occurred, otherwise calls the "failed" function and passes the given +// template and the error from "fn". func (c *controller) web( fn func(*http.Request) (interface{}, error), tmpl string, @@ -116,6 +134,8 @@ func (c *controller) web( } } +// writeJSON sets the header for the response and writes the JSON encoding of the given "content". +// It logs out an error message in case of an error. func writeJSON(rw http.ResponseWriter, content interface{}, code int) { rw.Header().Set("Content-type", "application/json; charset=utf-8") rw.Header().Set("X-Content-Type-Options", "nosniff") From c43d690b572027027cc29e7e61757f5a634af610 Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Mon, 7 Feb 2022 13:23:09 +0100 Subject: [PATCH 10/17] Fix Typo --- cmd/csaf_provider/actions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/csaf_provider/actions.go b/cmd/csaf_provider/actions.go index a4fc0a5..8d53083 100644 --- a/cmd/csaf_provider/actions.go +++ b/cmd/csaf_provider/actions.go @@ -129,7 +129,7 @@ func (c *controller) tlpParam(r *http.Request) (tlp, error) { return "", fmt.Errorf("unsupported TLP type '%s'", t) } -// creates calls the "ensureFolders" functions to create the directories and files. +// create calls the "ensureFolders" functions to create the directories and files. // It returns a struct by success, otherwise an error. func (c *controller) create(*http.Request) (interface{}, error) { if err := ensureFolders(c.cfg); err != nil { From 27f1aa5461e791c23a417567a93320657e2cf82c Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 7 Feb 2022 17:37:01 +0100 Subject: [PATCH 11/17] Started with loading provider-metadata.json from a list of possible locations. --- cmd/csaf_checker/processor.go | 120 ++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 3e8d403..47978d6 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -43,6 +43,7 @@ type processor struct { redirects map[string]string noneTLS map[string]struct{} alreadyChecked map[string]whereType + pmdURL string pmd256 []byte pmd interface{} keys []*crypto.KeyRing @@ -119,6 +120,7 @@ func (p *processor) clean() { for k := range p.alreadyChecked { delete(p.alreadyChecked, k) } + p.pmdURL = "" p.pmd256 = nil p.pmd = nil p.keys = nil @@ -131,6 +133,7 @@ func (p *processor) clean() { p.badIndices = nil p.badChanges = nil } + func (p *processor) run(reporters []reporter, domains []string) (*Report, error) { var report Report @@ -607,7 +610,7 @@ func (p *processor) checkChanges(base string, mask whereType) error { func (p *processor) processROLIEFeeds(domain string, feeds [][]csaf.Feed) error { - base, err := url.Parse("https://" + domain + "/.well-known/csaf/") + base, err := url.Parse(p.pmdURL) if err != nil { return err } @@ -654,7 +657,10 @@ func (p *processor) checkCSAFs(domain string) error { } // No rolie feeds - base := "https://" + domain + "/.well-known/csaf" + base, err := basePath(p.pmdURL) + if err != nil { + return err + } if err := p.checkIndex(base, indexMask); err != nil && err != errContinue { return err @@ -701,49 +707,91 @@ func (p *processor) checkMissing(string) error { return nil } -func (p *processor) checkProviderMetadata(domain string) error { +var providerMetadataLocations = [...]string{ + ".well-known/csaf", + "security/data/csaf", + "advisories/csaf", + "security/csaf", +} + +// locateProviderMetadata searches for provider-metadata.json at various +// locations mentioned in "7.1.7 Requirement 7: provider-metadata.json". +func (p *processor) locateProviderMetadata( + domain string, + found func(string, io.Reader) error, +) error { client := p.httpClient() - url := "https://" + domain + "/.well-known/csaf/provider-metadata.json" + for _, loc := range providerMetadataLocations { + url := "https://" + domain + "/" + loc + res, err := client.Get(url) + if err != nil { + continue + } + if res.StatusCode != http.StatusOK { + continue + } + if res.Header.Get("Content-Type") != "application/json" { + continue + } + + if err := func() error { + defer res.Body.Close() + return found(url, res.Body) + }(); err != nil { + if err == errContinue { + continue + } + return err + } + break + } + + // TODO: Read from security.txt + + return nil +} + +func (p *processor) checkProviderMetadata(domain string) error { use(&p.badProviderMetadatas) - res, err := client.Get(url) - if err != nil { - p.badProviderMetadata("Fetching %s: %v.", url, err) - return errStop + found := func(url string, content io.Reader) error { + + // Calculate checksum for later comparison. + hash := sha256.New() + + tee := io.TeeReader(content, hash) + if err := json.NewDecoder(tee).Decode(&p.pmd); err != nil { + p.badProviderMetadata("%s: Decoding JSON failed: %v", url, err) + return errContinue + } + + p.pmd256 = hash.Sum(nil) + + errors, err := csaf.ValidateProviderMetadata(p.pmd) + if err != nil { + return err + } + if len(errors) > 0 { + p.badProviderMetadata("%s: Validating against JSON schema failed:", url) + for _, msg := range errors { + p.badProviderMetadata(strings.ReplaceAll(msg, `%`, `%%`)) + } + return errStop + } + p.pmdURL = url + return nil } - if res.StatusCode != http.StatusOK { - p.badProviderMetadata("Fetching %s failed. Status code: %d (%s)", - url, res.StatusCode, res.Status) - return errStop - } - - // Calculate checksum for later comparison. - hash := sha256.New() - - if err := func() error { - defer res.Body.Close() - tee := io.TeeReader(res.Body, hash) - return json.NewDecoder(tee).Decode(&p.pmd) - }(); err != nil { - p.badProviderMetadata("Decoding JSON failed: %v", err) - return errStop - } - - p.pmd256 = hash.Sum(nil) - - errors, err := csaf.ValidateProviderMetadata(p.pmd) - if err != nil { + if err := p.locateProviderMetadata(domain, found); err != nil { return err } - if len(errors) > 0 { - p.badProviderMetadata("Validating against JSON schema failed:") - for _, msg := range errors { - p.badProviderMetadata(strings.ReplaceAll(msg, `%`, `%%`)) - } + + if p.pmdURL == "" { + p.badProviderMetadata("No provider-metadata.json found.") + return errStop } return nil } @@ -851,7 +899,7 @@ func (p *processor) checkPGPKeys(domain string) error { client := p.httpClient() - base, err := url.Parse("https://" + domain + "/.well-known/csaf/provider-metadata.json") + base, err := url.Parse(p.pmdURL) if err != nil { return err } From b894950b63935a20e39461feab34ddedd00d139d Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 7 Feb 2022 20:12:32 +0100 Subject: [PATCH 12/17] Load location of provider-metadata.json from security.txt --- cmd/csaf_checker/processor.go | 79 +++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 47978d6..49e4d62 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "net/url" "regexp" @@ -723,34 +724,86 @@ func (p *processor) locateProviderMetadata( client := p.httpClient() - for _, loc := range providerMetadataLocations { - url := "https://" + domain + "/" + loc + tryURL := func(url string) (bool, error) { res, err := client.Get(url) - if err != nil { - continue - } - if res.StatusCode != http.StatusOK { - continue - } - if res.Header.Get("Content-Type") != "application/json" { - continue + if err != nil || res.StatusCode != http.StatusOK || + res.Header.Get("Content-Type") != "application/json" { + // ignore this as it is expected. + return false, nil } if err := func() error { defer res.Body.Close() return found(url, res.Body) }(); err != nil { + return false, err + } + return true, nil + } + + for _, loc := range providerMetadataLocations { + url := "https://" + domain + "/" + loc + ok, err := tryURL(url) + if err != nil { if err == errContinue { continue } return err } - break + if ok { + return nil + } } - // TODO: Read from security.txt + // Read from security.txt - return nil + path := "https://" + domain + "/.well-known/security.txt" + res, err := client.Get(path) + if err != nil { + return err + } + + if res.StatusCode != http.StatusOK { + return err + } + + loc, err := func() (string, error) { + defer res.Body.Close() + return extractProviderURL(res.Body) + }() + + if err != nil { + log.Printf("error: %v\n", err) + return nil + } + + if loc != "" { + if _, err = tryURL(loc); err == errContinue { + err = nil + } + } + + return err +} + +func extractProviderURL(r io.Reader) (string, error) { + sc := bufio.NewScanner(r) + const csaf = "CSAF:" + + for sc.Scan() { + line := sc.Text() + if strings.HasPrefix(line, csaf) { + line = strings.TrimSpace(line[len(csaf):]) + if !strings.HasPrefix(line, "https://") { + return "", errors.New("CASF: found in security.txt, but does not start with https://") + } + return line, nil + } + } + if err := sc.Err(); err != nil { + return "", err + } + return "", nil } func (p *processor) checkProviderMetadata(domain string) error { From 65605b292878fbb778764806e24de94c7472abec Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Tue, 8 Feb 2022 17:31:49 +0100 Subject: [PATCH 13/17] Improve Makefile * Change mechanics to use a variable to indicate if a tag build is wanted. Add ability to specify tag explictely. * Use variables to have only one build recipe. * Place binaries in target specific bin- directories. * Do not return to main branch as it may not be the original branch we were on. Turn this into a warning. --- Makefile | 69 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 58cc0c5..d241d99 100644 --- a/Makefile +++ b/Makefile @@ -1,44 +1,49 @@ -# Simple Make file to build csaf_distribution components +# Makefile to build csaf_distribution components -SHELL=/bin/bash +SHELL = /bin/bash BUILD = go build -MKDIR = mkdir -p bin +MKDIR = mkdir -p -.PHONY: build build_win build_tag clean +.PHONY: build build_linux build_win tag_checked_out mostlyclean all: - @echo choose a target from: build build_linux build_win build_tag clean + @echo choose a target from: build build_linux build_win mostlyclean + @echo prepend \`make BUILDTAG=1\` to checkout the highest git tag before building + @echo or set BUILDTAG to a specific tag -# Build the binaries for GNU/linux and place them under bin/ directory. -build_linux: - @$(MKDIR) - @echo "Bulding binaries for GNU/Linux ..." - @$(BUILD) -o ./bin/ -v ./cmd/... - -# Build the binaries for windows (cross build) and place them under bin/ directory. -build_win: - @$(MKDIR) - @echo "Bulding binaries for windows (cross build) ..." - @env GOARCH=amd64 GOOS=windows $(BUILD) -o ./bin/ -v ./cmd/... - -# Build the binaries for both GNU/linux and Windows and place them under bin/ directory. +# Build all binaries build: build_linux build_win -# Build the binaries from the latest github tag. -TAG = $(shell git tag --sort=-version:refname | head -n 1) -build_tag: -ifeq ($(TAG),) - @echo "No Tag found" -else - @git checkout -q tags/${TAG}; - @echo $(buildMsg) - @$(BUILD) -o ./bin/ -v ./cmd/...; - @env GOOS=windows $(BUILD) -o ./ -v ./cmd/... - @git checkout -q main +# if BUILDTAG == 1 set it to the highest git tag +ifeq ($(strip $(BUILDTAG)),1) +override BUILDTAG = $(shell git tag --sort=-version:refname | head -n 1) endif -# Remove bin/ directory -clean: - @rm -rf bin/ +ifdef BUILDTAG +# add the git tag checkout to the requirements of our build targets +build_linux build_win: tag_checked_out +endif + +tag_checked_out: + $(if $(strip $(BUILDTAG)),,$(error no git tag found)) + git checkout -q tags/${BUILDTAG} + @echo Don\'t forget that we are in checked out tag $(BUILDTAG) now. +# Build binaries and place them under bin-$(GOOS)-$(GOARCH) +# Using 'Target-specific Variable Values' to specify the build target system + +GOARCH = amd64 +build_linux: GOOS = linux +build_win: GOOS = windows + +build_linux build_win: + $(eval BINDIR = bin-$(GOOS)-$(GOARCH)/ ) + $(MKDIR) $(BINDIR) + env GOARCH=$(GOARCH) GOOS=$(GOOS) $(BUILD) -o $(BINDIR) -v ./cmd/... + + +# Remove bin-*-* directories +mostlyclean: + rm -rf ./bin-*-* + @echo Files in \`go env GOCACHE\` remain. From 566e4260df4c6ade88f3951d53f06ac63e0d3fbc Mon Sep 17 00:00:00 2001 From: Fadi Abbud <39081670+Fadiabb@users.noreply.github.com> Date: Wed, 9 Feb 2022 15:52:05 +0100 Subject: [PATCH 14/17] Add License header to Makefile and adjust the README (#53) * Add License header to Makefile and adjust the README --- Makefile | 8 ++++++++ README.md | 7 +++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d241d99..c761729 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,11 @@ +# This file is Free Software under the MIT License +# without warranty, see README.md and LICENSES/MIT.txt for details. +# +# SPDX-License-Identifier: MIT +# +# SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) +# Software-Engineering: 2021 Intevation GmbH +# # Makefile to build csaf_distribution components SHELL = /bin/bash diff --git a/README.md b/README.md index 1497e5f..8541dab 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,12 @@ - Build For GNU/Linux System: `make build_linux` - Build For Windows System (cross build): `make build_win` - Build For both linux and windows: `make build` - - Build from the last github-tag: `make build_tag` + - Build from a specific github tag by passing the intended tag to the `BUILDTAG` variable. + E.g. `make BUILDTAG=v1.0.0 build` or `make BUILDTAG=1 build_linux`. + The special value `1` means checking out the highest github tag for the build. + - Remove the generated binaries und their directories: `make mostlyclean` -These places the binaries under `bin/` directory. +Binaries will be places in directories named like `bin-linux-amd64/` and `bin-windows-amd64/`. - [Install](http://nginx.org/en/docs/install.html) **nginx** - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) From e259a5878bffd1a9b3f2753db798ba67652ee245 Mon Sep 17 00:00:00 2001 From: Fadi Abbud Date: Wed, 9 Feb 2022 16:22:31 +0100 Subject: [PATCH 15/17] (minor) Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8541dab..02ce36a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The special value `1` means checking out the highest github tag for the build. - Remove the generated binaries und their directories: `make mostlyclean` -Binaries will be places in directories named like `bin-linux-amd64/` and `bin-windows-amd64/`. +Binaries will be placed in directories named like `bin-linux-amd64/` and `bin-windows-amd64/`. - [Install](http://nginx.org/en/docs/install.html) **nginx** - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) From 6a106640c68c2a42ea7ba71b6cd55a5dd6fe2dea Mon Sep 17 00:00:00 2001 From: Fadi Abbud <39081670+Fadiabb@users.noreply.github.com> Date: Mon, 14 Feb 2022 12:39:40 +0100 Subject: [PATCH 16/17] Improve docs: add instructions to install TLS cert for nginx * Add instructions for installing a TLS server certificate on nginx * Fix link to nginx in README.md * List all three ways to get a webserver TLS certificate. With some hints on which to chose for which purpose. * Do not add CSR instructions, because they can change over time and each CA may have slightly different requirements. * Add a hint about setting protocol selection. * Fix typo in provider-setup.md --- README.md | 3 +- docs/install-server-certificate.md | 72 ++++++++++++++++++++++++++++++ docs/provider-setup.md | 2 +- 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 docs/install-server-certificate.md diff --git a/README.md b/README.md index 02ce36a..9cfaba2 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Binaries will be placed in directories named like `bin-linux-amd64/` and `bin-windows-amd64/`. -- [Install](http://nginx.org/en/docs/install.html) **nginx** +- [Install](https://nginx.org/en/docs/install.html) **nginx** +- To install server certificate on nginx see [docs/install-server-certificate.md](docs/install-server-certificate.md) - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) ## csaf_uploader diff --git a/docs/install-server-certificate.md b/docs/install-server-certificate.md new file mode 100644 index 0000000..94b0340 --- /dev/null +++ b/docs/install-server-certificate.md @@ -0,0 +1,72 @@ +# Configure TLS Certificate for HTTPS + +## Get a webserver TLS certificate + +There are three ways to get a TLS certificate for your HTTPS server: + 1. Get it from a certificate provider who will run a certificate + authority (CA) and also offers + [extended validation](https://en.wikipedia.org/wiki/Extended_Validation_Certificate) (EV) + for the certificate. This will cost a fee. + If possible, create the private key yourself, + then send a Certificate Signing Request (CSR). + Overall follow the documentation of the CA operator. + 2. Get a domain validated TLS certificate via + [Let's encrypt](https://letsencrypt.org/) without a fee. + See their instruction, e.g. + [certbot for nignx on Ubuntu](https://certbot.eff.org/instructions?ws=nginx&os=ubuntufocal). + 3. Run your own little CA. Which has the major drawback that someone + will have to import the root certificate in the webbrowsers manually. + Suitable for development purposes. + +To decide between 1. and 2. you will need to weight the extra +efforts and costs of the level of extended validation against +a bit of extra trust for the security advisories +that will be served under the domain. + + +## Install the files for ngnix + +Place the certificates on the server machine. +This includes the certificate for your webserver, the intermediate +certificates and the root certificate. The latter may already be on your +machine as part of the trust anchors for webbrowsers. + +Follow the [nginx documentation](https://docs.nginx.com/nginx/admin-guide/security-controls/terminating-ssl-http/) +to further configure TLS with your private key and the certificates. + +We recommend to + * restrict the TLS protocol version and ciphers following a current + recommendation (e.g. [BSI-TR-02102-2](https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/Publications/TechGuidelines/TG02102/BSI-TR-02102-2.html)). + + +### Example configuration + +Assuming the relevant server block is in `/etc/nginx/sites-enabled/default`, +change the `listen` configuration and add options so nginx +finds your your private key and the certificate chain. + +```nginx +server { + listen 443 ssl http2 default_server; # ipv4 + listen [::]:443 ssl http2 default_server; # ipv6 + server_name www.example.com + + ssl_certificate /etc/ssl/{domainName}.pem; # or bundle.crt + ssl_certificate_key /etc/ssl/{domainName}.key"; + + ssl_protocols TLSv1.2 TLSv1.3; + # Other Config + # ... +} +``` + +Replace `{domainName}` with the name for your certificate in the example. + +Reload or restart nginx to apply the changes (e.g. `systemctl reload nginx` +on Debian or Ubuntu.) + +Technical hints: + * When allowing or requiring `TLSv1.3` webbrowsers like +Chromium (seen with version 98) may have higher requirements +on the server certificates they allow, +otherwise they do not connect with `ERR_SSL_KEY_USAGE_INCOMPATIBLE`. diff --git a/docs/provider-setup.md b/docs/provider-setup.md index da47fca..d737f03 100644 --- a/docs/provider-setup.md +++ b/docs/provider-setup.md @@ -7,7 +7,7 @@ The following instructions are for an Debian 11 server setup. ```(shell) apt-get install nginx fcgiwrap cp /usr/share/doc/fcgiwrap/examples/nginx.conf /etc/nginx/fcgiwrap.conf -systemctl status fcgiwrap.servic +systemctl status fcgiwrap.service systemctl status fcgiwrap.socket systemctl is-enabled fcgiwrap.service systemctl is-enabled fcgiwrap.socket From 78a04ab081e4d549a7cf35a61c0cfbec74f3e611 Mon Sep 17 00:00:00 2001 From: Fadi Abbud <39081670+Fadiabb@users.noreply.github.com> Date: Wed, 16 Feb 2022 16:32:58 +0100 Subject: [PATCH 17/17] Improve uploader documentation (#38) * improves #33 --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9cfaba2..927c0bd 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,19 @@ **WIP**: A proof of concept for a CSAF trusted provider, checker and aggregator. - ## Setup - A recent version of **Go** (1.17+) should be installed. [Go installation](https://go.dev/doc/install) - Clone the repository `git clone https://github.com/csaf-poc/csaf_distribution.git ` -- Build Go components - Makefile supplies the following targets: +- Build Go components Makefile supplies the following targets: - Build For GNU/Linux System: `make build_linux` - Build For Windows System (cross build): `make build_win` - Build For both linux and windows: `make build` - - Build from a specific github tag by passing the intended tag to the `BUILDTAG` variable. - E.g. `make BUILDTAG=v1.0.0 build` or `make BUILDTAG=1 build_linux`. - The special value `1` means checking out the highest github tag for the build. + - Build from a specific github tag by passing the intended tag to the `BUILDTAG` variable. + E.g. `make BUILDTAG=v1.0.0 build` or `make BUILDTAG=1 build_linux`. + The special value `1` means checking out the highest github tag for the build. - Remove the generated binaries und their directories: `make mostlyclean` Binaries will be placed in directories named like `bin-linux-amd64/` and `bin-windows-amd64/`. @@ -26,6 +24,7 @@ Binaries will be placed in directories named like `bin-linux-amd64/` and `bin-wi - To configure nginx see [docs/provider-setup.md](docs/provider-setup.md) ## csaf_uploader + csaf_uploader is a command line tool that uploads CSAF documents to the trusted provider (CSAF_Provider). Following options are supported: @@ -34,13 +33,14 @@ Following options are supported: | -a, --action=[upload\|create] | Action to perform (default: upload) | | -u, --url=URL | URL of the CSAF provider (default:https://localhost/cgi-bin/csaf_provider.go) | | -t, --tlp=[csaf\|white\|green\|amber\|red] | TLP of the feed (default: csaf) | -| -x, --external-signed | CASF files are signed externally. | +| -x, --external-signed | CASF files are signed externally. Assumes .asc files beside CSAF files | | -k, --key=KEY-FILE | OpenPGP key to sign the CSAF files | | -p, --password=PASSWORD | Authentication password for accessing the CSAF provider | | -P, --passphrase=PASSPHRASE | Passphrase to unlock the OpenPGP key | | -i, --password-interactive | Enter password interactively | | -I, --passphrase-interacive | Enter passphrase interactively | | -c, --config=INI-FILE | Path to config ini file | +| --insecure | Do not check TSL certificates from provider | | -h, --help | Show help | E.g. creating the initial directiories and files