Merge pull request #64 from csaf-poc/factor-out-summary-extraction

Factor out summary extraction from advisories.

cmd/csaf_checker/processor.go

@@ -11,7 +11,6 @@ package main
 import (
 	"bufio"
 	"bytes"
-	"context"
 	"crypto/sha256"
 	"crypto/sha512"
 	"crypto/tls"
@@ -29,8 +28,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/PaesslerAG/gval"
-	"github.com/PaesslerAG/jsonpath"
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 
 	"github.com/csaf-poc/csaf_distribution/csaf"
@@ -58,8 +55,7 @@ type processor struct {
 	badChanges           []string
 	badFolders           []string
 
-	builder gval.Language
-	exprs   map[string]gval.Evaluable
+	expr *util.PathEval
 }
 
 type reporter interface {
@@ -110,8 +106,7 @@ func newProcessor(opts *options) *processor {
 	return &processor{
 		opts:           opts,
 		alreadyChecked: map[string]whereType{},
-		builder:        gval.Full(jsonpath.Language()),
-		exprs:          map[string]gval.Evaluable{},
+		expr:           util.NewPathEval(),
 	}
 }
 
@@ -178,21 +173,6 @@ func (p *processor) checkDomain(domain string) error {
 	return nil
 }
 
-func (p *processor) jsonPath(expr string, doc interface{}) (interface{}, error) {
-	if doc == nil {
-		return nil, errors.New("no document to extract data from")
-	}
-	eval := p.exprs[expr]
-	if eval == nil {
-		var err error
-		if eval, err = p.builder.NewEvaluable(expr); err != nil {
-			return nil, err
-		}
-		p.exprs[expr] = eval
-	}
-	return eval(context.Background(), doc)
-}
-
 func (p *processor) checkTLS(u string) {
 	if p.noneTLS == nil {
 		p.noneTLS = map[string]struct{}{}
@@ -359,7 +339,7 @@ func (p *processor) integrity(
 		// Check if file is in the right folder.
 		use(&p.badFolders)
 
-		if date, err := p.jsonPath(
+		if date, err := p.expr.Eval(
 			`$.document.tracking.initial_release_date`, doc); err != nil {
 			p.badFolder(
 				"Extracting 'initial_release_date' from %s failed: %v", u, err)
@@ -638,7 +618,7 @@ func (p *processor) processROLIEFeeds(domain string, feeds [][]csaf.Feed) error
 
 func (p *processor) checkCSAFs(domain string) error {
 	// Check for ROLIE
-	rolie, err := p.jsonPath("$.distributions[*].rolie.feeds", p.pmd)
+	rolie, err := p.expr.Eval("$.distributions[*].rolie.feeds", p.pmd)
 	if err != nil {
 		return err
 	}
@@ -931,7 +911,7 @@ func (p *processor) checkPGPKeys(domain string) error {
 
 	use(&p.badPGPs)
 
-	src, err := p.jsonPath("$.pgp_keys", p.pmd)
+	src, err := p.expr.Eval("$.pgp_keys", p.pmd)
 	if err != nil {
 		p.badPGP("No PGP keys found: %v.", err)
 		return errContinue

cmd/csaf_provider/actions.go

@@ -167,7 +167,7 @@ func (c *controller) upload(r *http.Request) (interface{}, error) {
 		}
 	}
 
-	ex, err := newExtraction(content)
+	ex, err := csaf.NewAdvisorySummary(util.NewPathEval(), content)
 	if err != nil {
 		return nil, err
 	}
@@ -179,7 +179,7 @@ func (c *controller) upload(r *http.Request) (interface{}, error) {
 
 	// Extract real TLP from document.
 	if t == tlpCSAF {
-		if t = tlp(strings.ToLower(ex.tlpLabel)); !t.valid() || t == tlpCSAF {
+		if t = tlp(strings.ToLower(ex.TLPLabel)); !t.valid() || t == tlpCSAF {
 			return nil, fmt.Errorf(
 				"valid TLP label missing in document (found '%s')", t)
 		}
@@ -239,20 +239,20 @@ func (c *controller) upload(r *http.Request) (interface{}, error) {
 
 			rolie.Feed.Updated = csaf.TimeStamp(time.Now())
 
-			year := strconv.Itoa(ex.initialReleaseDate.Year())
+			year := strconv.Itoa(ex.InitialReleaseDate.Year())
 
 			csafURL := c.cfg.Domain +
 				"/.well-known/csaf/" + ts + "/" + year + "/" + newCSAF
 
-			e := rolie.EntryByID(ex.id)
+			e := rolie.EntryByID(ex.ID)
 			if e == nil {
-				e = &csaf.Entry{ID: ex.id}
+				e = &csaf.Entry{ID: ex.ID}
 				rolie.Feed.Entry = append(rolie.Feed.Entry, e)
 			}
 
-			e.Titel = ex.title
-			e.Published = csaf.TimeStamp(ex.initialReleaseDate)
-			e.Updated = csaf.TimeStamp(ex.currentReleaseDate)
+			e.Titel = ex.Title
+			e.Published = csaf.TimeStamp(ex.InitialReleaseDate)
+			e.Updated = csaf.TimeStamp(ex.CurrentReleaseDate)
 			e.Link = []csaf.Link{{
 				Rel:  "self",
 				HRef: csafURL,
@@ -265,8 +265,8 @@ func (c *controller) upload(r *http.Request) (interface{}, error) {
 				Type: "application/json",
 				Src:  csafURL,
 			}
-			if ex.summary != "" {
-				e.Summary = &csaf.Summary{Content: ex.summary}
+			if ex.Summary != "" {
+				e.Summary = &csaf.Summary{Content: ex.Summary}
 			} else {
 				e.Summary = nil
 			}
@@ -302,7 +302,7 @@ func (c *controller) upload(r *http.Request) (interface{}, error) {
 
 			if err := updateIndices(
 				folder, filepath.Join(year, newCSAF),
-				ex.currentReleaseDate,
+				ex.CurrentReleaseDate,
 			); err != nil {
 				return err
 			}
@@ -313,9 +313,9 @@ func (c *controller) upload(r *http.Request) (interface{}, error) {
 				warn("Publisher in provider metadata is not initialized. Forgot to configure?")
 				if c.cfg.DynamicProviderMetaData {
 					warn("Taking publisher from CSAF")
-					pmd.Publisher = ex.publisher
+					pmd.Publisher = ex.Publisher
 				}
-			case !pmd.Publisher.Equals(ex.publisher):
+			case !pmd.Publisher.Equals(ex.Publisher):
 				warn("Publishers in provider metadata and CSAF do not match.")
 			}
 
@@ -334,7 +334,7 @@ func (c *controller) upload(r *http.Request) (interface{}, error) {
 		Error       error    `json:"-"`
 	}{
 		Name:        newCSAF,
-		ReleaseDate: ex.currentReleaseDate.Format(dateFormat),
+		ReleaseDate: ex.CurrentReleaseDate.Format(dateFormat),
 		Warnings:    warnings,
 	}
 

csaf/summary.go

@@ -6,17 +6,12 @@
 // SPDX-FileCopyrightText: 2021 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
 // Software-Engineering: 2021 Intevation GmbH <https://intevation.de>
 
-package main
+package csaf
 
 import (
-	"context"
 	"errors"
 	"time"
 
-	"github.com/PaesslerAG/gval"
-	"github.com/PaesslerAG/jsonpath"
-
-	"github.com/csaf-poc/csaf_distribution/csaf"
 	"github.com/csaf-poc/csaf_distribution/util"
 )
 
@@ -30,39 +25,39 @@ const (
 	summaryExpr            = `$.document.notes[? @.category=="summary" || @.type=="summary"].text`
 )
 
-type extraction struct {
-	id                 string
-	title              string
-	publisher          *csaf.Publisher
-	initialReleaseDate time.Time
-	currentReleaseDate time.Time
-	summary            string
-	tlpLabel           string
+// AdvisorySummary is a summary of some essentials of an CSAF advisory.
+type AdvisorySummary struct {
+	ID                 string
+	Title              string
+	Publisher          *Publisher
+	InitialReleaseDate time.Time
+	CurrentReleaseDate time.Time
+	Summary            string
+	TLPLabel           string
 }
 
 type extractFunc func(string) (interface{}, error)
 
-func newExtraction(content interface{}) (*extraction, error) {
+// NewAdvisorySummary creates a summary from an advisory doc
+// with the help of an expression evaluator expr.
+func NewAdvisorySummary(
+	expr *util.PathEval,
+	doc interface{},
+) (*AdvisorySummary, error) {
 
-	builder := gval.Full(jsonpath.Language())
+	e := new(AdvisorySummary)
 
-	path := func(expr string) (interface{}, error) {
-		eval, err := builder.NewEvaluable(expr)
-		if err != nil {
-			return nil, err
-		}
-		return eval(context.Background(), content)
+	path := func(s string) (interface{}, error) {
+		return expr.Eval(s, doc)
 	}
 
-	e := new(extraction)
-
 	for _, fn := range []func(extractFunc) error{
-		extractText(idExpr, &e.id),
-		extractText(titleExpr, &e.title),
-		extractTime(currentReleaseDateExpr, &e.currentReleaseDate),
-		extractTime(initialReleaseDateExpr, &e.initialReleaseDate),
-		extractText(summaryExpr, &e.summary),
-		extractText(tlpLabelExpr, &e.tlpLabel),
+		extractText(idExpr, &e.ID),
+		extractText(titleExpr, &e.Title),
+		extractTime(currentReleaseDateExpr, &e.CurrentReleaseDate),
+		extractTime(initialReleaseDateExpr, &e.InitialReleaseDate),
+		extractText(summaryExpr, &e.Summary),
+		extractText(tlpLabelExpr, &e.TLPLabel),
 		e.extractPublisher,
 	} {
 		if err := fn(path); err != nil {
@@ -95,7 +90,7 @@ func extractTime(expr string, store *time.Time) func(extractFunc) error {
 		if !ok {
 			return errors.New("not a string")
 		}
-		date, err := time.Parse(dateFormat, text)
+		date, err := time.Parse(time.RFC3339, text)
 		if err == nil {
 			*store = date.UTC()
 		}
@@ -103,7 +98,7 @@ func extractTime(expr string, store *time.Time) func(extractFunc) error {
 	}
 }
 
-func (e *extraction) extractPublisher(path extractFunc) error {
+func (e *AdvisorySummary) extractPublisher(path extractFunc) error {
 	p, err := path(publisherExpr)
 	if err != nil {
 		return err
@@ -111,13 +106,13 @@ func (e *extraction) extractPublisher(path extractFunc) error {
 
 	// XXX: It's a bit cumbersome to serialize and deserialize
 	// it into our own structure.
-	publisher := new(csaf.Publisher)
+	publisher := new(Publisher)
 	if err := util.ReMarshalJSON(publisher, p); err != nil {
 		return err
 	}
 	if err := publisher.Validate(); err != nil {
 		return err
 	}
-	e.publisher = publisher
+	e.Publisher = publisher
 	return nil
 }

util/json.go

@@ -9,7 +9,12 @@
 package util
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
+
+	"github.com/PaesslerAG/gval"
+	"github.com/PaesslerAG/jsonpath"
 )
 
 // ReMarshalJSON transforms data from src to dst via JSON marshalling.
@@ -20,3 +25,34 @@ func ReMarshalJSON(dst, src interface{}) error {
 	}
 	return json.Unmarshal(intermediate, dst)
 }
+
+// PathEval is a helper to evaluate JSON paths on documents.
+type PathEval struct {
+	builder gval.Language
+	exprs   map[string]gval.Evaluable
+}
+
+// NewPathEval creates a new PathEval.
+func NewPathEval() *PathEval {
+	return &PathEval{
+		builder: gval.Full(jsonpath.Language()),
+		exprs:   map[string]gval.Evaluable{},
+	}
+}
+
+// Eval evalutes expression expr on document doc.
+// Returns the result of the expression.
+func (pe *PathEval) Eval(expr string, doc interface{}) (interface{}, error) {
+	if doc == nil {
+		return nil, errors.New("no document to extract data from")
+	}
+	eval := pe.exprs[expr]
+	if eval == nil {
+		var err error
+		if eval, err = pe.builder.NewEvaluable(expr); err != nil {
+			return nil, err
+		}
+		pe.exprs[expr] = eval
+	}
+	return eval(context.Background(), doc)
+}