diff --git a/cmd/csaf_checker/links.go b/cmd/csaf_checker/links.go index 07beada..f320931 100644 --- a/cmd/csaf_checker/links.go +++ b/cmd/csaf_checker/links.go @@ -10,11 +10,49 @@ package main import ( "io" + "net/http" + "net/url" "github.com/PuerkitoBio/goquery" ) -func linksOnPage(r io.Reader) ([]string, error) { +func (p *processor) linksOnPageURL(baseDir string) ([]string, error) { + + base, err := url.Parse(baseDir) + if err != nil { + return nil, err + } + + client := p.httpClient() + p.checkTLS(baseDir) + res, err := client.Get(baseDir) + + p.badDirListings.use() + + if err != nil { + p.badDirListings.add("Fetching %s failed: %v", base, err) + return nil, errContinue + } + if res.StatusCode != http.StatusOK { + p.badDirListings.add("Fetching %s failed. Status code %d (%s)", + base, res.StatusCode, res.Status) + return nil, errContinue + } + + defer res.Body.Close() + + // Links may be relative + return linksOnPage(res.Body, func(link string) (string, error) { + u, err := url.Parse(link) + if err != nil { + return "", err + } + return base.ResolveReference(u).String(), nil + }) + +} + +func linksOnPage(r io.Reader, resolve func(string) (string, error)) ([]string, error) { doc, err := goquery.NewDocumentFromReader(r) if err != nil { @@ -23,11 +61,16 @@ func linksOnPage(r io.Reader) ([]string, error) { var links []string - doc.Find("a").Each(func(i int, s *goquery.Selection) { + doc.Find("a").Each(func(_ int, s *goquery.Selection) { + if err != nil { + return + } if link, ok := s.Attr("href"); ok { - links = append(links, link) + if link, err = resolve(link); err == nil { + links = append(links, link) + } } }) - return links, nil + return links, err } diff --git a/cmd/csaf_checker/links_test.go b/cmd/csaf_checker/links_test.go index a8343ad..ad33aed 100644 --- a/cmd/csaf_checker/links_test.go +++ b/cmd/csaf_checker/links_test.go @@ -23,7 +23,10 @@ const page0 = ` func TestLinksOnPage(t *testing.T) { - links, err := linksOnPage(strings.NewReader(page0)) + links, err := linksOnPage( + strings.NewReader(page0), + func(s string) (string, error) { return s, nil }, + ) if err != nil { t.Fatal(err) } diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 56ec481..e98660a 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -59,6 +59,7 @@ type processor struct { badFolders topicMessages badWellknownMetadata topicMessages badDNSPath topicMessages + badDirListings topicMessages expr *util.PathEval } @@ -85,6 +86,8 @@ const ( rolieChangesMask indexMask changesMask + listingMask + rolieListingMask ) func (wt whereType) String() string { @@ -99,6 +102,10 @@ func (wt whereType) String() string { return "index.txt" case changesMask: return "changes.csv" + case listingMask: + return "directory listing" + case rolieListingMask: + return "directory listing [ROLIE]" default: var mixed []string for mask := rolieMask; mask <= changesMask; mask <<= 1 { @@ -157,6 +164,10 @@ func (p *processor) clean() { p.badSecurity.reset() p.badIndices.reset() p.badChanges.reset() + p.badFolders.reset() + p.badWellknownMetadata.reset() + p.badDNSPath.reset() + p.badDirListings.reset() } // run calls checkDomain function for each domain in the given "domains" parameter. @@ -483,6 +494,10 @@ func (p *processor) processROLIEFeed(feed string) error { return err } + if err := p.checkListing(base, rolieListingMask); err != nil && err != errContinue { + return err + } + return nil } @@ -652,6 +667,21 @@ func (p *processor) checkCSAFs(domain string) error { return err } + if err := p.checkListing(base, listingMask); err != nil && err != errContinue { + return err + } + + return nil +} + +func (p *processor) checkListing(base string, mask whereType) error { + links, err := p.linksOnPageURL(base) + if err != nil { + return err + } + for _, link := range links { + p.markChecked(link, mask) + } return nil } @@ -673,7 +703,7 @@ func (p *processor) checkMissing(string) error { for _, f := range files { v := p.alreadyChecked[f] var where []string - for mask := rolieMask; mask <= changesMask; mask <<= 1 { + for mask := rolieMask; mask <= rolieListingMask; mask <<= 1 { if maxMask&mask == mask { var in string if v&mask == mask { diff --git a/cmd/csaf_checker/reporters.go b/cmd/csaf_checker/reporters.go index 6abd5eb..c3c6b44 100644 --- a/cmd/csaf_checker/reporters.go +++ b/cmd/csaf_checker/reporters.go @@ -186,10 +186,17 @@ func (r *changesReporter) report(p *processor, domain *Domain) { req.Messages = p.badChanges } -func (r *directoryListingsReporter) report(_ *processor, domain *Domain) { - // TODO: Implement me! +func (r *directoryListingsReporter) report(p *processor, domain *Domain) { req := r.requirement(domain) - _ = req + if !p.badDirListings.used() { + req.message("No directory listings checked.") + return + } + if len(p.badDirListings) == 0 { + req.message("All directory listings are valid.") + return + } + req.Messages = p.badDirListings } func (r *integrityReporter) report(p *processor, domain *Domain) {