Merge pull request #649 from gocsaf/url-join

Use JoinPath

cmd/csaf_checker/links.go

@@ -9,6 +9,7 @@
 package main
 
 import (
+	"github.com/gocsaf/csaf/v3/internal/misc"
 	"io"
 	"net/http"
 	"net/url"
@@ -93,7 +94,7 @@ func (pgs pages) listed(
 				return err
 			}
 			// Links may be relative
-			abs := baseURL.ResolveReference(u).String()
+			abs := misc.JoinURL(baseURL, u).String()
 			content.links.Add(abs)
 			return nil
 		})

cmd/csaf_checker/processor.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/gocsaf/csaf/v3/internal/misc"
 	"io"
 	"log"
 	"net/http"
@@ -635,7 +636,6 @@ func (p *processor) integrity(
 	if err != nil {
 		return err
 	}
-	makeAbs := makeAbsolute(b)
 	client := p.httpClient()
 
 	var data bytes.Buffer
@@ -646,9 +646,8 @@ func (p *processor) integrity(
 			lg(ErrorType, "Bad URL %s: %v", f, err)
 			continue
 		}
-		fp = makeAbs(fp)
 
-		u := b.ResolveReference(fp).String()
+		u := misc.JoinURL(b, fp).String()
 
 		// Should this URL be ignored?
 		if p.cfg.ignoreURL(u) {
@@ -780,8 +779,7 @@ func (p *processor) integrity(
 				lg(ErrorType, "Bad URL %s: %v", x.url(), err)
 				continue
 			}
-			hu = makeAbs(hu)
+			hashFile := misc.JoinURL(b, hu).String()
-			hashFile := b.ResolveReference(hu).String()
 
 			p.checkTLS(hashFile)
 			if res, err = client.Get(hashFile); err != nil {
@@ -830,8 +828,7 @@ func (p *processor) integrity(
 			lg(ErrorType, "Bad URL %s: %v", f.SignURL(), err)
 			continue
 		}
-		su = makeAbs(su)
+		sigFile := misc.JoinURL(b, su).String()
-		sigFile := b.ResolveReference(su).String()
 		p.checkTLS(sigFile)
 
 		p.badSignatures.use()
@@ -1378,7 +1375,7 @@ func (p *processor) checkSecurityFolder(folder string) string {
 		return err.Error()
 	}
 
-	u = base.ResolveReference(up).String()
+	u = misc.JoinURL(base, up).String()
 	p.checkTLS(u)
 	if res, err = client.Get(u); err != nil {
 		return fmt.Sprintf("Cannot fetch %s from security.txt: %v", u, err)
@@ -1530,6 +1527,7 @@ func (p *processor) checkPGPKeys(_ string) error {
 	if err != nil {
 		return err
 	}
+	base.Path = ""
 
 	for i := range keys {
 		key := &keys[i]
@@ -1543,7 +1541,7 @@ func (p *processor) checkPGPKeys(_ string) error {
 			continue
 		}
 
-		u := base.ResolveReference(up).String()
+		u := misc.JoinURL(base, up).String()
 		p.checkTLS(u)
 
 		res, err := client.Get(u)

cmd/csaf_checker/roliecheck.go

@@ -10,6 +10,7 @@ package main
 
 import (
 	"errors"
+	"github.com/gocsaf/csaf/v3/internal/misc"
 	"net/http"
 	"net/url"
 	"sort"
@@ -221,6 +222,7 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 	if err != nil {
 		return err
 	}
+	base.Path = ""
 	p.badROLIEFeed.use()
 
 	advisories := map[*csaf.Feed][]csaf.AdvisoryFile{}
@@ -237,7 +239,7 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 				p.badProviderMetadata.error("Invalid URL %s in feed: %v.", *feed.URL, err)
 				continue
 			}
-			feedBase := base.ResolveReference(up)
+			feedBase := misc.JoinURL(base, up)
 			feedURL := feedBase.String()
 			p.checkTLS(feedURL)
 
@@ -270,7 +272,7 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 				continue
 			}
 
-			feedURL := base.ResolveReference(up)
+			feedURL := misc.JoinURL(base, up)
 			feedBase, err := util.BaseURL(feedURL)
 			if err != nil {
 				p.badProviderMetadata.error("Bad base path: %v", err)
@@ -290,7 +292,7 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 			// TODO: Issue a warning if we want check AMBER+ without an
 			// authorizing client.
 
-			if err := p.integrity(files, feedBase, rolieMask, p.badProviderMetadata.add); err != nil {
+			if err := p.integrity(files, base.String(), rolieMask, p.badProviderMetadata.add); err != nil {
 				if err != errContinue {
 					return err
 				}
@@ -325,7 +327,7 @@ func (p *processor) processROLIEFeeds(feeds [][]csaf.Feed) error {
 				continue
 			}
 
-			feedBase := base.ResolveReference(up)
+			feedBase := misc.JoinURL(base, up)
 			makeAbs := makeAbsolute(feedBase)
 			label := defaults(feed.TLPLabel, csaf.TLPLabelUnlabeled)
 

cmd/csaf_downloader/downloader.go

@@ -229,6 +229,7 @@ func (d *downloader) download(ctx context.Context, domain string) error {
 	if err != nil {
 		return fmt.Errorf("invalid URL '%s': %v", lpmd.URL, err)
 	}
+	base.Path = ""
 
 	expr := util.NewPathEval()
 
@@ -343,7 +344,7 @@ func (d *downloader) loadOpenPGPKeys(
 			continue
 		}
 
-		u := base.ResolveReference(up).String()
+		u := base.JoinPath(up.Path).String()
 
 		res, err := client.Get(u)
 		if err != nil {

csaf/advisories.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"encoding/csv"
 	"fmt"
+	"github.com/gocsaf/csaf/v3/internal/misc"
 	"io"
 	"log/slog"
 	"net/http"
@@ -281,7 +282,7 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 			slog.Error("Invalid URL in feed", "feed", *feed.URL, "err", err)
 			continue
 		}
-		feedURL := afp.base.ResolveReference(up)
+		feedURL := misc.JoinURL(afp.base, up)
 		slog.Info("Got feed URL", "feed", feedURL)
 
 		fb, err := util.BaseURL(feedURL)
@@ -294,6 +295,7 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 			slog.Error("Cannot parse feed base URL", "url", fb, "err", err)
 			continue
 		}
+		feedBaseURL.Path = ""
 
 		res, err := afp.client.Get(feedURL.String())
 		if err != nil {
@@ -325,7 +327,7 @@ func (afp *AdvisoryFileProcessor) processROLIE(
 				slog.Error("Invalid URL", "url", u, "err", err)
 				return ""
 			}
-			return feedBaseURL.ResolveReference(p).String()
+			return misc.JoinURL(feedBaseURL, p).String()
 		}
 
 		rfeed.Entries(func(entry *Entry) {

internal/misc/url.go

@@ -0,0 +1,21 @@
+// This file is Free Software under the Apache-2.0 License
+// without warranty, see README.md and LICENSES/Apache-2.0.txt for details.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// SPDX-FileCopyrightText: 2025 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
+// Software-Engineering: 2025 Intevation GmbH <https://intevation.de>
+
+package misc
+
+import "net/url"
+
+// JoinURL joins the two URLs while preserving the query and fragment part of the latter.
+func JoinURL(baseURL *url.URL, relativeURL *url.URL) *url.URL {
+	u := baseURL.JoinPath(relativeURL.Path)
+	u.RawQuery = relativeURL.RawQuery
+	u.RawFragment = relativeURL.RawFragment
+	// Enforce https, this is required if the base url was only a domain
+	u.Scheme = "https"
+	return u
+}