Handle flaky vulnerability tests (#11262)

- Refactored some of the vulnerabilities tests to help with flakiness.
- Don't load NVD assets if local assets have a timestamp of today.
This commit is contained in:
Juan Fernandez 2023-04-21 19:37:29 -04:00 committed by GitHub
parent cda88779c1
commit d353a3deb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 62 additions and 42 deletions

View File

@ -0,0 +1 @@
Fall back to a previous NVD release if the asset we want is not in the latest release.

View File

@ -487,12 +487,13 @@ func TestScanVulnerabilities(t *testing.T) {
}, nil
}
vulnPath := t.TempDir()
vulnPath := filepath.Join("..", "..", "server", "vulnerabilities", "testdata")
config := config.VulnerabilitiesConfig{
DatabasesPath: vulnPath,
Periodicity: 10 * time.Second,
CurrentInstanceChecks: "auto",
DisableDataSync: true,
}
ctx = license.NewContext(ctx, &fleet.LicenseInfo{Tier: fleet.TierPremium})

View File

@ -28,7 +28,6 @@ func DownloadAndExtract(client *http.Client, u *url.URL, path string) error {
}
func download(client *http.Client, u *url.URL, path string, extract bool) error {
// atomically write to file
dir, file := filepath.Split(path)
if dir == "" {

View File

@ -10,6 +10,7 @@ import (
"path/filepath"
"regexp"
"strings"
"time"
"github.com/doug-martin/goqu/v9"
"github.com/fleetdm/fleet/v4/pkg/download"
@ -31,23 +32,36 @@ const (
var cpeDBRegex = regexp.MustCompile(`^cpe-.*\.sqlite\.gz$`)
func GetLatestGithubNVDRelease() (*github.RepositoryRelease, error) {
githubClient := github.NewClient(fleethttp.NewGithubClient())
releases, _, err := githubClient.Repositories.ListReleases(
context.Background(), owner, repo, &github.ListOptions{Page: 0, PerPage: 10},
// GetGithubNVDAsset looks at the last 10 releases and returns the first (release, asset) pair that
// matches pred
func GetGithubNVDAsset(pred func(rel *github.ReleaseAsset) bool) (*github.RepositoryRelease, *github.ReleaseAsset, error) {
ghClient := github.NewClient(fleethttp.NewGithubClient())
releases, _, err := ghClient.Repositories.ListReleases(
context.Background(),
owner,
repo,
&github.ListOptions{Page: 0, PerPage: 10},
)
if err != nil {
return nil, err
return nil, nil, err
}
for _, release := range releases {
// skip draft releases
if !release.GetDraft() {
return release, nil
if release.GetDraft() {
continue
}
for _, asset := range release.Assets {
if pred(asset) {
return release, asset, nil
}
}
return nil, errors.New("no nvd release found")
}
return nil, nil, errors.New("no nvd release found")
}
// DownloadCPEDB downloads the CPE database to the given vulnPath. If cpeDBURL is empty, attempts to download it
@ -56,32 +70,34 @@ func DownloadCPEDBFromGithub(vulnPath string, cpeDBURL string) error {
path := filepath.Join(vulnPath, cpeDBFilename)
if cpeDBURL == "" {
release, err := GetLatestGithubNVDRelease()
if err != nil {
return err
}
stat, err := os.Stat(path)
switch {
case errors.Is(err, os.ErrNotExist):
// okay
case err != nil:
return err
default:
if stat.ModTime().After(release.CreatedAt.Time) {
case stat.ModTime().Truncate(24 * time.Hour).Equal(time.Now().Truncate(24 * time.Hour)):
// Vulnerability assets are published once per day - if the asset in question has a
// mod date of 'today', then we can assume that is already up to day so there's nothing
// else to do.
return nil
}
rel, asset, err := GetGithubNVDAsset(func(asset *github.ReleaseAsset) bool {
return cpeDBRegex.MatchString(asset.GetName())
})
if err != nil {
return err
}
if asset == nil {
return errors.New("failed to find cpe database in nvd release")
}
if stat != nil && stat.ModTime().After(rel.CreatedAt.Time) {
// file is newer than release, do nothing
return nil
}
}
for _, asset := range release.Assets {
if cpeDBRegex.MatchString(asset.GetName()) {
cpeDBURL = asset.GetBrowserDownloadURL()
break
}
}
if cpeDBURL == "" {
return errors.New("failed to find cpe database in nvd release")
}
}
u, err := url.Parse(cpeDBURL)

View File

@ -8,10 +8,12 @@ import (
"os"
"path/filepath"
"regexp"
"time"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/google/go-github/v37/github"
)
const cpeTranslationsFilename = "cpe_translations.json"
@ -37,32 +39,32 @@ func DownloadCPETranslationsFromGithub(vulnPath string, cpeTranslationsURL strin
path := filepath.Join(vulnPath, cpeTranslationsFilename)
if cpeTranslationsURL == "" {
release, err := GetLatestGithubNVDRelease()
if err != nil {
return err
}
stat, err := os.Stat(path)
switch {
case errors.Is(err, os.ErrNotExist):
// okay
case err != nil:
return err
default:
if stat.ModTime().After(release.CreatedAt.Time) {
case stat.ModTime().Truncate(24 * time.Hour).Equal(time.Now().Truncate(24 * time.Hour)):
// Vulnerability assets are published once per day - if the asset in question has a
// mod date of 'today', then we can assume that is already up to day.
return nil
}
release, asset, err := GetGithubNVDAsset(func(asset *github.ReleaseAsset) bool {
return cpeTranslationsFilename == asset.GetName()
})
if err != nil {
return err
}
if asset == nil {
return errors.New("failed to find cpe translations in nvd release")
}
if stat != nil && stat.ModTime().After(release.CreatedAt.Time) {
// file is newer than release, do nothing
return nil
}
}
for _, asset := range release.Assets {
if cpeTranslationsFilename == asset.GetName() {
cpeTranslationsURL = asset.GetBrowserDownloadURL()
break
}
}
if cpeTranslationsURL == "" {
return errors.New("failed to find cpe translations in nvd release")
}
}
u, err := url.Parse(cpeTranslationsURL)

View File

@ -2,11 +2,12 @@ package nvd
import (
"context"
"github.com/fleetdm/fleet/v4/server/contexts/license"
"path/filepath"
"strings"
"testing"
"github.com/fleetdm/fleet/v4/server/contexts/license"
"github.com/fleetdm/fleet/v4/pkg/nettest"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mock"
@ -65,7 +66,7 @@ func TestLoadCVEMeta(t *testing.T) {
require.Equal(t, false, *meta.CISAKnownExploit)
meta = metaMap["CVE-2022-22587"]
require.Equal(t, (*float64)(nil), meta.CVSSScore)
require.Equal(t, float64(9.8), *meta.CVSSScore)
require.Equal(t, float64(0.01843), *meta.EPSSProbability)
require.Equal(t, true, *meta.CISAKnownExploit)
}

View File

Binary file not shown.