gocsaf/cmd/csaf_checker/checks.go

// This file is Free Software under the MIT License
// without warranty, see README.md and LICENSES/MIT.txt for details.
//
// SPDX-License-Identifier: MIT
//
// SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2021 Intevation GmbH <https://intevation.de>

package main

import (
	"bufio"
	"bytes"
	"context"
	"crypto/sha256"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strings"

	"github.com/PaesslerAG/gval"
	"github.com/PaesslerAG/jsonpath"
	"github.com/ProtonMail/gopenpgp/v2/crypto"
	"github.com/csaf-poc/csaf_distribution/csaf"
	"github.com/csaf-poc/csaf_distribution/util"
)

type processor struct {
	opts      *options
	redirects map[string]string
	noneTLS   map[string]struct{}
	pmd256    []byte
	pmd       interface{}
	builder   gval.Language
	keys      []*crypto.Key
}

type check interface {
	executionOrder() int
	run(*processor, string) error
	report(*processor, *Domain)
}

func newProcessor(opts *options) *processor {
	return &processor{
		opts:      opts,
		redirects: map[string]string{},
		noneTLS:   map[string]struct{}{},
		builder:   gval.Full(jsonpath.Language()),
	}
}

func (p *processor) clean() {
	for k := range p.redirects {
		delete(p.redirects, k)
	}
	for k := range p.noneTLS {
		delete(p.noneTLS, k)
	}
	p.pmd256 = nil
	p.pmd = nil
	p.keys = nil
}

func (p *processor) run(checks []check, domains []string) (*Report, error) {

	var report Report

	execs := make([]check, len(checks))
	copy(execs, checks)
	sort.SliceStable(execs, func(i, j int) bool {
		return execs[i].executionOrder() < execs[j].executionOrder()
	})

	for _, d := range domains {
		for _, ch := range execs {
			if err := ch.run(p, d); err != nil {
				return nil, err
			}
		}
		domain := &Domain{Name: d}
		for _, ch := range checks {
			ch.report(p, domain)
		}
		report.Domains = append(report.Domains, domain)
		p.clean()
	}

	return &report, nil
}

func (p *processor) jsonPath(expr string) (interface{}, error) {
	if p.pmd == nil {
		return nil, errors.New("no provider metadata loaded")
	}
	eval, err := p.builder.NewEvaluable(expr)
	if err != nil {
		return nil, err
	}
	return eval(context.Background(), p.pmd)
}

func (p *processor) checkTLS(u string) {
	if x, err := url.Parse(u); err == nil && x.Scheme != "https" {
		p.noneTLS[u] = struct{}{}
	}
}

func (p *processor) checkRedirect(r *http.Request, via []*http.Request) error {

	var path strings.Builder
	for i, v := range via {
		if i > 0 {
			path.WriteString(", ")
		}
		path.WriteString(v.URL.String())
	}
	url := r.URL.String()
	p.checkTLS(url)
	p.redirects[url] = path.String()

	if len(via) > 10 {
		return errors.New("Too many redirections")
	}
	return nil
}

func (p *processor) httpClient() *http.Client {
	client := http.Client{
		CheckRedirect: p.checkRedirect,
	}

	if p.opts.Insecure {
		client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		}
	}

	return &client
}

func (p *processor) integrity(
	files []string,
	base string,
	lg func(string, ...interface{}),
) error {
	b, err := url.Parse(base)
	if err != nil {
		return err
	}
	client := p.httpClient()
	for _, f := range files {
		fp, err := url.Parse(f)
		if err != nil {
			return err
		}
		u := b.ResolveReference(fp).String()
		p.checkTLS(u)
		res, err := client.Get(u)
		if err != nil {
			lg("Fetching %s failed: %v.", u, err)
			continue
		}
		if res.StatusCode != http.StatusOK {
			lg("Fetchinf %s failed: Status code %d (%s)",
				u, res.StatusCode, res.Status)
			continue
		}
		data, err := func() ([]byte, error) {
			defer res.Body.Close()
			return io.ReadAll(res.Body)
		}()
		if err != nil {
			lg("Reading %s failed: %v", u, err)
			continue
		}
		var doc interface{}
		if err := json.Unmarshal(data, &doc); err != nil {
			lg("Failed to unmarshal %s: %v", u, err)
			continue
		}
		errors, err := csaf.ValidateCSAF(doc)
		if err != nil {
			lg("Failed to validate %s: %v", u, err)
			continue
		}
		if len(errors) > 0 {
			lg("CSAF file %s has %d validation errors.", u, len(errors))
		}
		// TODO: Implement check sums
		// TODO: Implement signature check.
	}
	return nil
}

type baseCheck struct {
	exec        int
	num         int
	description string
	messages    []string
}

type tlsCheck struct {
	baseCheck
}

type redirectsCheck struct {
	baseCheck
}

type providerMetadataCheck struct {
	baseCheck
}

type securityCheck struct {
	baseCheck
}

type wellknownMetadataCheck struct {
	baseCheck
}

type dnsPathCheck struct {
	baseCheck
}

type oneFolderPerYearCheck struct {
	baseCheck
}

type indexCheck struct {
	baseCheck
}

type changesCheck struct {
	baseCheck
}

type directoryListingsCheck struct {
	baseCheck
}

type integrityCheck struct {
	baseCheck
}

type signaturesCheck struct {
	baseCheck
}

type publicPGPKeyCheck struct {
	baseCheck
}

func (bc *baseCheck) executionOrder() int {
	return bc.exec
}

func (bc *baseCheck) run(*processor, string) error {
	return nil
}

func (bc *baseCheck) report(_ *processor, domain *Domain) {
	req := &Requirement{
		Num:         bc.num,
		Description: bc.description,
		Messages:    bc.messages,
	}
	domain.Requirements = append(domain.Requirements, req)
}

func (bc *baseCheck) add(messages ...string) {
	bc.messages = append(bc.messages, messages...)
}

func (bc *baseCheck) sprintf(format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	bc.messages = append(bc.messages, msg)
}

func (bc *baseCheck) ok(message string) bool {
	k := len(bc.messages) == 0
	if k {
		bc.messages = []string{message}
	}
	return k
}

func (tc *tlsCheck) run(p *processor, domain string) error {
	if len(p.noneTLS) == 0 {
		tc.add("All tested URLs were https.")
	} else {
		urls := make([]string, len(p.noneTLS))
		var i int
		for k := range p.noneTLS {
			urls[i] = k
			i++
		}
		sort.Strings(urls)
		tc.add("Following none https URLs were used:")
		tc.add(urls...)
	}
	return nil
}

func (rc *redirectsCheck) run(p *processor, domain string) error {
	if len(p.redirects) == 0 {
		rc.add("No redirections found.")
	} else {
		keys := make([]string, len(p.redirects))
		var i int
		for k := range p.redirects {
			keys[i] = k
			i++
		}
		sort.Strings(keys)
		for i, k := range keys {
			keys[i] = fmt.Sprintf("Redirect %s: %s", k, p.redirects[k])
		}
		rc.baseCheck.messages = keys
	}
	return nil
}

func (pmdc *providerMetadataCheck) run(p *processor, domain string) error {
	url := "https://" + domain + "/.well-known/csaf/provider-metadata.json"
	client := p.httpClient()
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	res, err := client.Do(req)
	if err != nil {
		pmdc.sprintf("Fetching provider metadata failed: %s.", err.Error())
		return nil
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		pmdc.sprintf("Status: %d (%s).", res.StatusCode, res.Status)
	}

	// Calculate checksum for later comparison.
	var buf bytes.Buffer
	if _, err := io.Copy(&buf, res.Body); err != nil {
		return err
	}
	data := buf.Bytes()
	h := sha256.New()
	if _, err := h.Write(data); err != nil {
		return err
	}
	p.pmd256 = h.Sum(nil)

	if err := json.NewDecoder(bytes.NewReader(data)).Decode(&p.pmd); err != nil {
		pmdc.sprintf("Decoding JSON failed: %s.", err.Error())
	}
	errors, err := csaf.ValidateProviderMetadata(p.pmd)
	if err != nil {
		return err
	}
	if len(errors) > 0 {
		pmdc.add("Validating against JSON schema failed:")
		pmdc.add(errors...)
	}

	pmdc.ok("No problems with provider metadata.")
	return nil
}

func (sc *securityCheck) run(p *processor, domain string) error {
	path := "https://" + domain + "/.well-known/security.txt"
	client := p.httpClient()
	req, err := http.NewRequest(http.MethodGet, path, nil)
	if err != nil {
		return err
	}
	res, err := client.Do(req)
	if err != nil {
		return err
	}
	if res.StatusCode != http.StatusOK {
		sc.sprintf("Fetching security failed. Status code %d (%s)", res.StatusCode, res.Status)
		return nil
	}
	u, err := func() (string, error) {
		defer res.Body.Close()
		lines := bufio.NewScanner(res.Body)
		for lines.Scan() {
			line := lines.Text()
			if strings.HasPrefix(line, "CSAF:") {
				return strings.TrimSpace(line[6:]), nil
			}
		}
		return "", lines.Err()
	}()
	if err != nil {
		sc.sprintf("Error while reading security.txt: %s", err.Error())
	}
	if u == "" {
		sc.add("No CSAF line found in security.txt.")
		return nil
	}

	// Try to load
	up, err := url.Parse(u)
	if err != nil {
		sc.sprintf("CSAF URL '%s' invalid: %s.", u, err.Error())
		return nil
	}

	base, err := url.Parse("https://" + domain + "/.well-known/")
	if err != nil {
		return err
	}
	ur := base.ResolveReference(up)
	u = ur.String()
	p.checkTLS(u)
	if req, err = http.NewRequest(http.MethodGet, u, nil); err != nil {
		return err
	}
	if res, err = client.Do(req); err != nil {
		sc.sprintf("Cannot fetch %s from security.txt: %v", u, err)
		return nil
	}
	if res.StatusCode != http.StatusOK {
		sc.sprintf("Fetching %s failed. Status code %d (%s).",
			u, res.StatusCode, res.Status)
		return nil
	}
	defer res.Body.Close()
	// Compare checksums to already read provider-metadata.json.
	h := sha256.New()
	if _, err := io.Copy(h, res.Body); err != nil {
		sc.sprintf("Reading %s failed: %v.", u, err)
		return nil
	}

	if !bytes.Equal(h.Sum(nil), p.pmd256) {
		sc.sprintf(
			"Content of %s from security.txt is not identical to .well-known/csaf/provider-metadata.json", u)
	}

	sc.ok("Valid CSAF line in security.txt found.")

	return nil
}

func (wmdc *wellknownMetadataCheck) run(*processor, string) error {
	// TODO: Implement me!
	return nil
}

func (dpc *dnsPathCheck) run(*processor, string) error {
	// TODO: Implement me!
	return nil
}

func basePath(p string) (string, error) {
	u, err := url.Parse(p)
	if err != nil {
		return "", err
	}
	ep := u.EscapedPath()
	if idx := strings.LastIndexByte(ep, '/'); idx != -1 {
		ep = ep[:idx]
	}
	user := u.User.String()
	if user != "" {
		user += "@"
	}
	return u.Scheme + "://" + user + u.Host + "/" + ep, nil
}

func (ofpyc *oneFolderPerYearCheck) processFeed(
	p *processor,
	feed string,
) error {
	client := p.httpClient()
	res, err := client.Get(feed)
	if err != nil {
		ofpyc.sprintf("Cannot fetch feed %s: %v.", feed, err)
		return nil
	}
	if res.StatusCode != http.StatusOK {
		ofpyc.sprintf("Fetching %s failed. Status code %d (%s)",
			feed, res.StatusCode, res.Status)
		return nil
	}
	rfeed, err := func() (*csaf.ROLIEFeed, error) {
		defer res.Body.Close()
		return csaf.LoadROLIEFeed(res.Body)
	}()
	if err != nil {
		ofpyc.sprintf("Loading ROLIE feed failed: %v.", err)
		return nil
	}
	base, err := basePath(feed)
	if err != nil {
		return err
	}

	// Extract the CSAF files from feed.
	var files []string
	for _, f := range rfeed.Entry {
		for i := range f.Link {
			files = append(files, f.Link[i].HRef)
		}
	}
	return p.integrity(files, base, ofpyc.sprintf)
}

func (ofpyc *oneFolderPerYearCheck) processFeeds(
	p *processor,
	domain string,
	feeds [][]csaf.Feed,
) error {
	base, err := url.Parse("https://" + domain + "/.well-known/csaf/")
	if err != nil {
		return err
	}
	for i := range feeds {
		for j := range feeds[i] {
			feed := &feeds[i][j]
			if feed.URL == nil {
				continue
			}
			up, err := url.Parse(string(*feed.URL))
			if err != nil {
				ofpyc.sprintf("Invalid URL %s in feed: %v.", *feed.URL, err)
				continue
			}
			feedURL := base.ResolveReference(up).String()
			p.checkTLS(feedURL)
			if err := ofpyc.processFeed(p, feedURL); err != nil {
				return err
			}
		}
	}
	return nil
}

func (ofpyc *oneFolderPerYearCheck) run(p *processor, domain string) error {
	// Check for ROLIE
	rolie, err := p.jsonPath("$.distributions[*].rolie.feeds")
	if err != nil {
		return err
	}

	fs, hasRolie := rolie.([]interface{})
	hasRolie = hasRolie && len(fs) > 0

	if hasRolie {
		var feeds [][]csaf.Feed
		if err := util.ReMarshalJSON(&feeds, rolie); err != nil {
			ofpyc.sprintf("ROLIE feeds are not compatible: %v.", err)
			return nil
		}
		if err := ofpyc.processFeeds(p, domain, feeds); err != nil {
			return err
		}
	} else {
		// No rolie feeds
		// TODO: Implement me!
	}

	return nil
}

func (ic *indexCheck) run(*processor, string) error {
	// TODO: Implement me!
	return nil
}

func (cc *changesCheck) run(*processor, string) error {
	// TODO: Implement me!
	return nil
}

func (dlc *directoryListingsCheck) run(*processor, string) error {
	// TODO: Implement me!
	return nil
}

func (ic *integrityCheck) run(*processor, string) error {
	// TODO: Implement me!
	return nil
}

func (sc *signaturesCheck) run(*processor, string) error {
	// TODO: Implement me!
	return nil
}

func reserialize(dst, src interface{}) error {
	s, err := json.Marshal(src)
	if err != nil {
		return err
	}
	return json.Unmarshal(s, dst)
}

func (ppkc *publicPGPKeyCheck) run(p *processor, domain string) error {

	src, err := p.jsonPath("$.pgp_keys")
	if err != nil {
		ppkc.sprintf("No PGP keys found: %v.", err)
		return nil
	}

	var keys []csaf.PGPKey
	if err := util.ReMarshalJSON(&keys, src); err != nil {
		ppkc.sprintf("PGP keys invalid: %v.", err)
		return nil
	}

	if len(keys) == 0 {
		ppkc.add("No PGP keys found.")
		return nil
	}

	// Try to load

	client := p.httpClient()

	base, err := url.Parse("https://" + domain + "/.well-known/csaf/provider-metadata.json")
	if err != nil {
		return err
	}

	for i := range keys {
		key := &keys[i]
		if key.URL == nil {
			ppkc.sprintf("Missing URL for fingerprint %x.", key.Fingerprint)
			continue
		}
		up, err := url.Parse(*key.URL)
		if err != nil {
			ppkc.sprintf("Invalid URL '%s': %v", *key.URL, err)
			continue
		}

		up = base.ResolveReference(up)
		u := up.String()
		p.checkTLS(u)

		req, err := http.NewRequest(http.MethodGet, u, nil)
		if err != nil {
			return err
		}
		res, err := client.Do(req)
		if err != nil {
			ppkc.sprintf("Fetching PGP key %s failed: %v.", u, err)
			continue
		}
		if res.StatusCode != http.StatusOK {
			ppkc.sprintf("Fetching PGP key %s status code: %d (%s)", u, res.StatusCode, res.Status)
			continue
		}

		ckey, err := func() (*crypto.Key, error) {
			defer res.Body.Close()
			return crypto.NewKeyFromArmoredReader(res.Body)
		}()

		if err != nil {
			ppkc.sprintf("Reading PGP key %s failed: %v", u, err)
			continue
		}

		if ckey.GetFingerprint() != string(key.Fingerprint) {
			ppkc.sprintf("Fingerprint of PGP key %s do not match remotely loaded.", u)
			continue
		}
		p.keys = append(p.keys, ckey)
	}

	if len(p.keys) == 0 {
		ppkc.add("No PGP keys loaded.")
		return nil
	}

	ppkc.ok(fmt.Sprintf("%d PGP key(s) loaded successfully.", len(p.keys)))

	return nil
}