From 36702ede8d864dcb7e2e4e08b15e7d861d6b5ad6 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Tue, 12 Apr 2022 14:48:15 -0400 Subject: [PATCH] Make recent vulnerabilities max age configurable. (#5081) --- ...ssue-jira-loadtest-add-recent-vuln-max-age | 1 + docs/Deploying/Configuration.md | 20 +++++++++++-- docs/Using-Fleet/Automations.md | 16 +++++------ server/config/config.go | 28 +++++++++++-------- server/vulnerabilities/cve.go | 7 ++--- server/vulnerabilities/cve_test.go | 5 ++-- 6 files changed, 46 insertions(+), 31 deletions(-) create mode 100644 changes/issue-jira-loadtest-add-recent-vuln-max-age diff --git a/changes/issue-jira-loadtest-add-recent-vuln-max-age b/changes/issue-jira-loadtest-add-recent-vuln-max-age new file mode 100644 index 000000000..1a3fd179d --- /dev/null +++ b/changes/issue-jira-loadtest-add-recent-vuln-max-age @@ -0,0 +1 @@ +* Add the `vulnerabilities.recent_vulnerability_max_age` configuration option. diff --git a/docs/Deploying/Configuration.md b/docs/Deploying/Configuration.md index 852e0a584..f0a6b7a25 100644 --- a/docs/Deploying/Configuration.md +++ b/docs/Deploying/Configuration.md @@ -47,7 +47,7 @@ The `fleet` binary contains several "commands". Similarly to how `git` has many - `fleet prepare db` - `fleet serve` - `fleet version` -- `fleet config_dump` +- `fleet config_dump` ### Options @@ -89,7 +89,7 @@ FLEET_LOGGING_JSON=true \ /usr/bin/fleet serve ``` -##### Using a YAML config file +##### Using a YAML config file ``` echo ' @@ -2176,7 +2176,20 @@ To download the data streams, you can use `fleetctl vulnerability-data-stream -- vulnerabilities: disable_data_sync: true ``` - + +##### recent_vulnerability_max_age + +Maximum age of a vulnerability (a CVE) to be considered "recent". The age is calculated based on the published date of the CVE in the [National Vulnerability Database](https://nvd.nist.gov/) (NVD). Recent vulnerabilities play a special role in Fleet's [automations](../Using-Fleet/Automations.md), as they are reported when discovered on a host if the vulnerabilities webhook or a vulnerability integration is enabled. + +- Default value: `720h` (30 days) +- Environment variable: `FLEET_VULNERABILITIES_RECENT_VULNERABILITY_MAX_AGE` +- Config file format: + + ``` + vulnerabilities: + recent_vulnerability_max_age: 48h + ``` + ##### Example YAML ```yaml @@ -2189,6 +2202,7 @@ spec: disable_data_sync: true ``` + #### GeoIP ##### database_path diff --git a/docs/Using-Fleet/Automations.md b/docs/Using-Fleet/Automations.md index ae032f49a..15246f809 100644 --- a/docs/Using-Fleet/Automations.md +++ b/docs/Using-Fleet/Automations.md @@ -2,7 +2,7 @@ You can configure automations in Fleet to send a webhook request if a certain condition is met. -[Vulnerability automations](#policy-automations) send a webhook request if a new vulnerability (CVE) is +[Vulnerability automations](#vulnerability-automations) send a webhook request if a new vulnerability (CVE) is detected on at least one host. [Policy automations](#policy-automations) send a webhook request if a policy is newly failing on at @@ -17,7 +17,7 @@ Vulnerability automations send a webhook request if a new vulnerability (CVE) is found on at least one host. > Note that a CVE is "new" if it was published to the national vulnerability (NVD) database within -> the last 2 days. +> the last 30 days (by default). Fleet sends these webhook requests once every hour. If two new vulnerabilities are detected within the hour, two @@ -54,10 +54,10 @@ POST https://server.com/example ## Policy automations Policy automations send a webhook request if a policy is newly failing on at -least one host. +least one host. > Note that a policy is "newly failing" if a host updated its response from "no response" to "failing" -> or from "passing" to "failing." +> or from "passing" to "failing." Fleet sends these webhook requests once per day. If two policies are newly failing within the day, two webhook requests are sent. This interval can be updated with the `webhook_settings.interval` @@ -117,9 +117,9 @@ POST https://server.com/example ```json { - "text": "More than X% of your hosts have not checked into Fleet - for more than X days. You’ve been sent this message - because the Host status webhook is enabeld in your Fleet + "text": "More than X% of your hosts have not checked into Fleet + for more than X days. You’ve been sent this message + because the Host status webhook is enabeld in your Fleet instance.", "data": { "unseen_hosts": 1, @@ -132,4 +132,4 @@ POST https://server.com/example To enable and configure host status automations, navigate to **Settings > Organization settings > Host status webhook** in the Fleet UI. - \ No newline at end of file + diff --git a/server/config/config.go b/server/config/config.go index 8850e7f16..ef6922fc9 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -216,12 +216,13 @@ type LicenseConfig struct { // VulnerabilitiesConfig defines configs related to vulnerability processing within Fleet. type VulnerabilitiesConfig struct { - DatabasesPath string `json:"databases_path" yaml:"databases_path"` - Periodicity time.Duration `json:"periodicity" yaml:"periodicity"` - CPEDatabaseURL string `json:"cpe_database_url" yaml:"cpe_database_url"` - CVEFeedPrefixURL string `json:"cve_feed_prefix_url" yaml:"cve_feed_prefix_url"` - CurrentInstanceChecks string `json:"current_instance_checks" yaml:"current_instance_checks"` - DisableDataSync bool `json:"disable_data_sync" yaml:"disable_data_sync"` + DatabasesPath string `json:"databases_path" yaml:"databases_path"` + Periodicity time.Duration `json:"periodicity" yaml:"periodicity"` + CPEDatabaseURL string `json:"cpe_database_url" yaml:"cpe_database_url"` + CVEFeedPrefixURL string `json:"cve_feed_prefix_url" yaml:"cve_feed_prefix_url"` + CurrentInstanceChecks string `json:"current_instance_checks" yaml:"current_instance_checks"` + DisableDataSync bool `json:"disable_data_sync" yaml:"disable_data_sync"` + RecentVulnerabilityMaxAge time.Duration `json:"recent_vulnerability_max_age" yaml:"recent_vulnerability_max_age"` } // UpgradesConfig defines configs related to fleet server upgrades. @@ -567,6 +568,8 @@ func (man Manager) addConfigs() { "Allows to manually select an instance to do the vulnerability processing.") man.addConfigBool("vulnerabilities.disable_data_sync", false, "Skips synchronizing data streams and expects them to be available in the databases_path.") + man.addConfigDuration("vulnerabilities.recent_vulnerability_max_age", 30*24*time.Hour, + "Maximum age of the published date of a vulnerability (CVE) to be considered 'recent'.") // Upgrades man.addConfigBool("upgrades.allow_missing_migrations", false, @@ -748,12 +751,13 @@ func (man Manager) LoadConfig() FleetConfig { Key: man.getConfigString("license.key"), }, Vulnerabilities: VulnerabilitiesConfig{ - DatabasesPath: man.getConfigString("vulnerabilities.databases_path"), - Periodicity: man.getConfigDuration("vulnerabilities.periodicity"), - CPEDatabaseURL: man.getConfigString("vulnerabilities.cpe_database_url"), - CVEFeedPrefixURL: man.getConfigString("vulnerabilities.cve_feed_prefix_url"), - CurrentInstanceChecks: man.getConfigString("vulnerabilities.current_instance_checks"), - DisableDataSync: man.getConfigBool("vulnerabilities.disable_data_sync"), + DatabasesPath: man.getConfigString("vulnerabilities.databases_path"), + Periodicity: man.getConfigDuration("vulnerabilities.periodicity"), + CPEDatabaseURL: man.getConfigString("vulnerabilities.cpe_database_url"), + CVEFeedPrefixURL: man.getConfigString("vulnerabilities.cve_feed_prefix_url"), + CurrentInstanceChecks: man.getConfigString("vulnerabilities.current_instance_checks"), + DisableDataSync: man.getConfigBool("vulnerabilities.disable_data_sync"), + RecentVulnerabilityMaxAge: man.getConfigDuration("vulnerabilities.recent_vulnerability_max_age"), }, Upgrades: UpgradesConfig{ AllowMissingMigrations: man.getConfigBool("upgrades.allow_missing_migrations"), diff --git a/server/vulnerabilities/cve.go b/server/vulnerabilities/cve.go index 8b6521014..ec7450b9c 100644 --- a/server/vulnerabilities/cve.go +++ b/server/vulnerabilities/cve.go @@ -59,10 +59,6 @@ const publishedDateFmt = "2006-01-02T15:04Z" // not quite RFC3339 var ( rxNVDCVEArchive = regexp.MustCompile(`nvdcve.*\.gz$`) - // max age to be considered a recent vulnerability (relative to NVD's published date) - // (a var to be able to change in tests) - recentVulnMaxAge = 30 * 24 * time.Hour - // this allows mocking the time package for tests, by default it is equivalent // to the time functions, e.g. theClock.Now() == time.Now(). theClock clock.Clock = clock.C @@ -124,7 +120,7 @@ func TranslateCPEToCVE( recentVulns = make(map[string][]string) } for _, file := range files { - err := checkCVEs(ctx, ds, logger, cpes, file, recentVulns) + err := checkCVEs(ctx, ds, logger, cpes, file, recentVulns, config.Vulnerabilities.RecentVulnerabilityMaxAge) if err != nil { return nil, err } @@ -140,6 +136,7 @@ func checkCVEs( cpes []*wfn.Attributes, file string, recentVulns map[string][]string, + recentVulnMaxAge time.Duration, ) error { dict, err := cvefeed.LoadJSONDictionary(file) diff --git a/server/vulnerabilities/cve_test.go b/server/vulnerabilities/cve_test.go index 0c0d21843..e17ace5c4 100644 --- a/server/vulnerabilities/cve_test.go +++ b/server/vulnerabilities/cve_test.go @@ -75,6 +75,7 @@ func TestTranslateCPEToCVE(t *testing.T) { err := SyncCVEData(tempDir, cfg) require.NoError(t, err) cfg.Vulnerabilities.DisableDataSync = true + cfg.Vulnerabilities.RecentVulnerabilityMaxAge = 365 * 24 * time.Hour for _, tt := range cvetests { t.Run(tt.cpe, func(t *testing.T) { @@ -110,9 +111,7 @@ func TestTranslateCPEToCVE(t *testing.T) { // consider recent vulnerabilities to be anything published in 2018 theClock = clock.NewMockClock(time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC)) - oldMaxAge := recentVulnMaxAge - recentVulnMaxAge = 365 * 24 * time.Hour - defer func() { recentVulnMaxAge = oldMaxAge; theClock = clock.C }() + defer func() { theClock = clock.C }() safeDS := &threadSafeDSMock{Store: ds}