SystemSh0cker/gocsaf
1
0
Fork 0
mirror of https://github.com/gocsaf/csaf.git synced 2025-12-22 18:15:42 +01:00
Code Issues Releases Activity Actions

If there are year folders in directory listings, fetch files from the extra level.

Browse source
This commit is contained in:
Sascha L. Teichmann 2022-05-17 15:58:34 +02:00
parent 29f26e0299
commit 131a7155fa
3 changed files with 85 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

100
cmd/csaf_checker/links.go
View file

@ -12,11 +12,14 @@ import (
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"regexp"
"strings" "strings"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
) )
var yearFolder = regexp.MustCompile(`.*/?\d{4}/?$`)
func (p *processor) linksOnPageURL(baseDir string) ([]string, error) { func (p *processor) linksOnPageURL(baseDir string) ([]string, error) {
base, err := url.Parse(baseDir) base, err := url.Parse(baseDir)
@ -40,42 +43,87 @@ func (p *processor) linksOnPageURL(baseDir string) ([]string, error) {
return nil, errContinue return nil, errContinue
} }
defer res.Body.Close() var (
subDirs []string
// Links may be relative files []string
return linksOnPage(res.Body, func(link string) (string, error) { )
u, err := url.Parse(link) if err := func() error {
if err != nil { defer res.Body.Close()
return "", err return linksOnPage(res.Body, func(link string) error {
} u, err := url.Parse(link)
return base.ResolveReference(u).String(), nil if err != nil {
}) return err
}
} // Links may be relative
abs := base.ResolveReference(u).String()
func linksOnPage(r io.Reader, resolve func(string) (string, error)) ([]string, error) { switch {
case yearFolder.MatchString(link):
doc, err := goquery.NewDocumentFromReader(r) subDirs = append(subDirs, abs)
if err != nil { case strings.HasSuffix(link, ".json"):
files = append(files, abs)
}
return nil
})
}(); err != nil {
return nil, err return nil, err
} }
var links []string // If we do not have sub folders, return links from this level.
if len(subDirs) == 0 {
return files, nil
}
// Descent into folders
for _, sub := range subDirs {
p.checkTLS(sub)
res, err := client.Get(sub)
if err != nil {
p.badDirListings.add("Fetching %s failed: %v", sub, err)
return nil, errContinue
}
if res.StatusCode != http.StatusOK {
p.badDirListings.add("Fetching %s failed. Status code %d (%s)",
base, res.StatusCode, res.Status)
return nil, errContinue
}
if err := func() error {
defer res.Body.Close()
return linksOnPage(res.Body, func(link string) error {
u, err := url.Parse(link)
if err != nil {
return err
}
// Links may be relative
abs := base.ResolveReference(u).String()
// Only collect json files in this sub folder
if strings.HasSuffix(link, ".json") {
files = append(files, abs)
}
return nil
})
}(); err != nil {
return nil, err
}
}
return files, nil
}
func linksOnPage(r io.Reader, visit func(string) error) error {
doc, err := goquery.NewDocumentFromReader(r)
if err != nil {
return err
}
doc.Find("a").Each(func(_ int, s *goquery.Selection) { doc.Find("a").Each(func(_ int, s *goquery.Selection) {
if err != nil { if err != nil {
return return
} }
if link, ok := s.Attr("href"); ok { if link, ok := s.Attr("href"); ok {
// Only care for JSON files here. err = visit(link)
if !strings.HasSuffix(link, ".json") {
return
}
if link, err = resolve(link); err == nil {
links = append(links, link)
}
} }
}) })
return links, err return err
} }

11
cmd/csaf_checker/links_test.go
View file

@ -24,9 +24,16 @@ const page0 = `<html>
func TestLinksOnPage(t *testing.T) { func TestLinksOnPage(t *testing.T) {
links, err := linksOnPage( var links []string
err := linksOnPage(
strings.NewReader(page0), strings.NewReader(page0),
func(s string) (string, error) { return s, nil }, func(s string) error {
if strings.HasSuffix(s, ".json") {
links = append(links, s)
}
return nil
},
) )
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

4
cmd/csaf_checker/processor.go
View file

@ -252,7 +252,7 @@ func (p *processor) checkRedirect(r *http.Request, via []*http.Request) error {
p.redirects[url] = path.String() p.redirects[url] = path.String()
if len(via) > 10 { if len(via) > 10 {
return errors.New("Too many redirections") return errors.New("too many redirections")
} }
return nil return nil
} }
@ -826,7 +826,7 @@ func (p *processor) extractProviderURL(r io.Reader) (string, error) {
return "", err return "", err
} }
if len(urls) == 0 { if len(urls) == 0 {
return "", errors.New("No provider-metadata.json found") return "", errors.New("no provider-metadata.json found")
} }
if len(urls) > 1 { if len(urls) > 1 {