Handle flaky vulnerability tests (#11262)

- Refactored some of the vulnerabilities tests to help with flakiness.
- Don't load NVD assets if local assets have a timestamp of today.

changes/bug-handle-missing-nvd-assets

@@ -0,0 +1 @@
+Fall back to a previous NVD release if the asset we want is not in the latest release.

cmd/fleet/serve_test.go

@@ -487,12 +487,13 @@ func TestScanVulnerabilities(t *testing.T) {
 		}, nil
 	}
 
-	vulnPath := t.TempDir()
+	vulnPath := filepath.Join("..", "..", "server", "vulnerabilities", "testdata")
 
 	config := config.VulnerabilitiesConfig{
 		DatabasesPath:         vulnPath,
 		Periodicity:           10 * time.Second,
 		CurrentInstanceChecks: "auto",
+		DisableDataSync:       true,
 	}
 
 	ctx = license.NewContext(ctx, &fleet.LicenseInfo{Tier: fleet.TierPremium})

pkg/download/download.go

@@ -28,7 +28,6 @@ func DownloadAndExtract(client *http.Client, u *url.URL, path string) error {
 }
 
 func download(client *http.Client, u *url.URL, path string, extract bool) error {
-
 	// atomically write to file
 	dir, file := filepath.Split(path)
 	if dir == "" {

server/vulnerabilities/nvd/cpe.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/doug-martin/goqu/v9"
 	"github.com/fleetdm/fleet/v4/pkg/download"
@@ -31,23 +32,36 @@ const (
 
 var cpeDBRegex = regexp.MustCompile(`^cpe-.*\.sqlite\.gz$`)
 
-func GetLatestGithubNVDRelease() (*github.RepositoryRelease, error) {
-	githubClient := github.NewClient(fleethttp.NewGithubClient())
-	releases, _, err := githubClient.Repositories.ListReleases(
-		context.Background(), owner, repo, &github.ListOptions{Page: 0, PerPage: 10},
+// GetGithubNVDAsset looks at the last 10 releases and returns the first (release, asset) pair that
+// matches pred
+func GetGithubNVDAsset(pred func(rel *github.ReleaseAsset) bool) (*github.RepositoryRelease, *github.ReleaseAsset, error) {
+	ghClient := github.NewClient(fleethttp.NewGithubClient())
+
+	releases, _, err := ghClient.Repositories.ListReleases(
+		context.Background(),
+		owner,
+		repo,
+		&github.ListOptions{Page: 0, PerPage: 10},
 	)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	for _, release := range releases {
 		// skip draft releases
-		if !release.GetDraft() {
-			return release, nil
+		if release.GetDraft() {
+			continue
 		}
+
+		for _, asset := range release.Assets {
+			if pred(asset) {
+				return release, asset, nil
+			}
+		}
+
 	}
 
-	return nil, errors.New("no nvd release found")
+	return nil, nil, errors.New("no nvd release found")
 }
 
 // DownloadCPEDB downloads the CPE database to the given vulnPath. If cpeDBURL is empty, attempts to download it
@@ -56,32 +70,34 @@ func DownloadCPEDBFromGithub(vulnPath string, cpeDBURL string) error {
 	path := filepath.Join(vulnPath, cpeDBFilename)
 
 	if cpeDBURL == "" {
-		release, err := GetLatestGithubNVDRelease()
-		if err != nil {
-			return err
-		}
 		stat, err := os.Stat(path)
 		switch {
 		case errors.Is(err, os.ErrNotExist):
 			// okay
 		case err != nil:
 			return err
-		default:
-			if stat.ModTime().After(release.CreatedAt.Time) {
-				// file is newer than release, do nothing
-				return nil
-			}
+		case stat.ModTime().Truncate(24 * time.Hour).Equal(time.Now().Truncate(24 * time.Hour)):
+			// Vulnerability assets are published once per day - if the asset in question has a
+			// mod date of 'today', then we can assume that is already up to day so there's nothing
+			// else to do.
+			return nil
 		}
 
-		for _, asset := range release.Assets {
-			if cpeDBRegex.MatchString(asset.GetName()) {
-				cpeDBURL = asset.GetBrowserDownloadURL()
-				break
-			}
+		rel, asset, err := GetGithubNVDAsset(func(asset *github.ReleaseAsset) bool {
+			return cpeDBRegex.MatchString(asset.GetName())
+		})
+		if err != nil {
+			return err
 		}
-		if cpeDBURL == "" {
+		if asset == nil {
 			return errors.New("failed to find cpe database in nvd release")
 		}
+		if stat != nil && stat.ModTime().After(rel.CreatedAt.Time) {
+			// file is newer than release, do nothing
+			return nil
+		}
+
+		cpeDBURL = asset.GetBrowserDownloadURL()
 	}
 
 	u, err := url.Parse(cpeDBURL)

server/vulnerabilities/nvd/cpe_translations.go

@@ -8,10 +8,12 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"time"
 
 	"github.com/fleetdm/fleet/v4/pkg/download"
 	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
 	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/google/go-github/v37/github"
 )
 
 const cpeTranslationsFilename = "cpe_translations.json"
@@ -37,32 +39,32 @@ func DownloadCPETranslationsFromGithub(vulnPath string, cpeTranslationsURL strin
 	path := filepath.Join(vulnPath, cpeTranslationsFilename)
 
 	if cpeTranslationsURL == "" {
-		release, err := GetLatestGithubNVDRelease()
-		if err != nil {
-			return err
-		}
 		stat, err := os.Stat(path)
 		switch {
 		case errors.Is(err, os.ErrNotExist):
 			// okay
 		case err != nil:
 			return err
-		default:
-			if stat.ModTime().After(release.CreatedAt.Time) {
-				// file is newer than release, do nothing
-				return nil
-			}
+		case stat.ModTime().Truncate(24 * time.Hour).Equal(time.Now().Truncate(24 * time.Hour)):
+			// Vulnerability assets are published once per day - if the asset in question has a
+			// mod date of 'today', then we can assume that is already up to day.
+			return nil
 		}
 
-		for _, asset := range release.Assets {
-			if cpeTranslationsFilename == asset.GetName() {
-				cpeTranslationsURL = asset.GetBrowserDownloadURL()
-				break
-			}
+		release, asset, err := GetGithubNVDAsset(func(asset *github.ReleaseAsset) bool {
+			return cpeTranslationsFilename == asset.GetName()
+		})
+		if err != nil {
+			return err
 		}
-		if cpeTranslationsURL == "" {
+		if asset == nil {
 			return errors.New("failed to find cpe translations in nvd release")
 		}
+		if stat != nil && stat.ModTime().After(release.CreatedAt.Time) {
+			// file is newer than release, do nothing
+			return nil
+		}
+		cpeTranslationsURL = asset.GetBrowserDownloadURL()
 	}
 
 	u, err := url.Parse(cpeTranslationsURL)

server/vulnerabilities/nvd/sync_test.go

@@ -2,11 +2,12 @@ package nvd
 
 import (
 	"context"
-	"github.com/fleetdm/fleet/v4/server/contexts/license"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/fleetdm/fleet/v4/server/contexts/license"
+
 	"github.com/fleetdm/fleet/v4/pkg/nettest"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/mock"
@@ -65,7 +66,7 @@ func TestLoadCVEMeta(t *testing.T) {
 	require.Equal(t, false, *meta.CISAKnownExploit)
 
 	meta = metaMap["CVE-2022-22587"]
-	require.Equal(t, (*float64)(nil), meta.CVSSScore)
+	require.Equal(t, float64(9.8), *meta.CVSSScore)
 	require.Equal(t, float64(0.01843), *meta.EPSSProbability)
 	require.Equal(t, true, *meta.CISAKnownExploit)
 }