mirror of
https://github.com/empayre/fleet.git
synced 2024-11-06 08:55:24 +00:00
Add enable scheduled query stats to fleet config (#4066)
* Add enable scheduled query stats to fleet config as well
* Add documentation
* Revert "Allow disabling scheduled query stats via app config (#4049)"
This reverts commit f98fd4d331.
* Add changes file
* Update ref
* Add missing docs
This commit is contained in:
Tomas Touceda
GitHub
parent
f855e45089
commit
11887f87f7
1
changes/add-enable-scheduled-query-stats-to-fleetconfig
Normal file
1
changes/add-enable-scheduled-query-stats-to-fleetconfig
Normal file
@ -0,0 +1 @@
|
||||
* Allow to disable scheduled query stats through fleet serve config
|
||||
@ -1 +0,0 @@
|
||||
* Allow disabling scheduled query stats via app config
|
||||
@ -398,7 +398,6 @@ spec:
|
||||
host_expiry_window: 0
|
||||
host_settings:
|
||||
enable_host_users: true
|
||||
enable_scheduled_query_stats: false
|
||||
enable_software_inventory: false
|
||||
org_info:
|
||||
org_logo_url: ""
|
||||
@ -450,7 +449,7 @@ spec:
|
||||
enable_vulnerabilities_webhook: false
|
||||
host_batch_size: 0
|
||||
`
|
||||
expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}}
|
||||
expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}}
|
||||
`
|
||||
|
||||
assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config"}))
|
||||
@ -468,7 +467,6 @@ spec:
|
||||
host_expiry_window: 0
|
||||
host_settings:
|
||||
enable_host_users: true
|
||||
enable_scheduled_query_stats: false
|
||||
enable_software_inventory: false
|
||||
license:
|
||||
expiration: "0001-01-01T00:00:00Z"
|
||||
@ -550,7 +548,7 @@ spec:
|
||||
enable_vulnerabilities_webhook: false
|
||||
host_batch_size: 0
|
||||
`
|
||||
expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}}
|
||||
expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}}
|
||||
`
|
||||
|
||||
assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config", "--include-server-config"}))
|
||||
|
||||
@ -6,7 +6,6 @@
|
||||
- [Enroll secrets](#enroll-secrets)
|
||||
- [Teams](#teams)
|
||||
- [Organization settings](#organization-settings)
|
||||
- [Host settings](#host-settings)
|
||||
|
||||
Entities in Fleet, such as queries, packs, labels, agent options, and enroll secrets, can be managed with configuration files in yaml syntax.
|
||||
|
||||
@ -500,4 +499,3 @@ later on processed by Fleet for different functionalities.
|
||||
|
||||
- `host_settings.enable_host_users`: boolean value that when enabled Fleet will send the query needed to gather user data
|
||||
- `host_settings.enable_software_inventory`: boolean value that when enabled Fleet will send the query needed to gather the list of software installed along with other metadata
|
||||
- `host_settings.enable_scheduled_query_stats`: boolean value that when enabled Fleet will send the query needed to gather statistics about query executions in a host
|
||||
|
||||
@ -769,6 +769,19 @@ How long invite tokens should be valid for.
|
||||
invite_token_validity_period: 1d
|
||||
```
|
||||
|
||||
##### app_enable_scheduled_query_stats
|
||||
|
||||
Determines whether Fleet gets scheduled query statistics from hosts or not.
|
||||
|
||||
- Default value: `true`
|
||||
- Environment variable: `FLEET_APP_ENABLE_SCHEDULED_QUERY_STATS`
|
||||
- Config file format:
|
||||
|
||||
```
|
||||
app:
|
||||
enable_scheduled_query_stats: true
|
||||
```
|
||||
|
||||
#### License
|
||||
|
||||
##### license_key
|
||||
|
||||
@ -93,6 +93,7 @@ type AuthConfig struct {
|
||||
type AppConfig struct {
|
||||
TokenKeySize int `yaml:"token_key_size"`
|
||||
InviteTokenValidityPeriod time.Duration `yaml:"invite_token_validity_period"`
|
||||
EnableScheduledQueryStats bool `yaml:"enable_scheduled_query_stats"`
|
||||
}
|
||||
|
||||
// SessionConfig defines configs related to user sessions
|
||||
@ -385,6 +386,8 @@ func (man Manager) addConfigs() {
|
||||
"Duration invite tokens remain valid (i.e. 1h)")
|
||||
man.addConfigInt("app.token_key_size", 24,
|
||||
"Size of generated tokens")
|
||||
man.addConfigBool("app.enable_scheduled_query_stats", true,
|
||||
"If true (default) it gets scheduled query stats from hosts")
|
||||
|
||||
// Session
|
||||
man.addConfigInt("session.key_size", 64,
|
||||
@ -610,6 +613,7 @@ func (man Manager) LoadConfig() FleetConfig {
|
||||
App: AppConfig{
|
||||
TokenKeySize: man.getConfigInt("app.token_key_size"),
|
||||
InviteTokenValidityPeriod: man.getConfigDuration("app.invite_token_validity_period"),
|
||||
EnableScheduledQueryStats: man.getConfigBool("app.enable_scheduled_query_stats"),
|
||||
},
|
||||
Session: SessionConfig{
|
||||
KeySize: man.getConfigInt("session.key_size"),
|
||||
|
||||
@ -280,7 +280,6 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) {
|
||||
require.Equal(t, 24*time.Hour, ac.WebhookSettings.Interval.Duration)
|
||||
require.False(t, ac.WebhookSettings.HostStatusWebhook.Enable)
|
||||
require.True(t, ac.HostSettings.EnableHostUsers)
|
||||
require.True(t, ac.HostSettings.EnableScheduledQueryStats)
|
||||
require.False(t, ac.HostSettings.EnableSoftwareInventory)
|
||||
|
||||
_, err = ds.writer.Exec(
|
||||
@ -294,6 +293,5 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) {
|
||||
|
||||
require.Equal(t, 12*time.Hour, ac.WebhookSettings.Interval.Duration)
|
||||
require.False(t, ac.HostSettings.EnableHostUsers)
|
||||
require.True(t, ac.HostSettings.EnableScheduledQueryStats)
|
||||
require.False(t, ac.HostSettings.EnableSoftwareInventory)
|
||||
}
|
||||
|
||||
@ -230,7 +230,6 @@ func (c *AppConfig) ApplyDefaultsForNewInstalls() {
|
||||
|
||||
func (c *AppConfig) ApplyDefaults() {
|
||||
c.HostSettings.EnableHostUsers = true
|
||||
c.HostSettings.EnableScheduledQueryStats = true
|
||||
c.WebhookSettings.Interval.Duration = 24 * time.Hour
|
||||
}
|
||||
|
||||
@ -256,10 +255,9 @@ type HostExpirySettings struct {
|
||||
}
|
||||
|
||||
type HostSettings struct {
|
||||
EnableHostUsers bool `json:"enable_host_users"`
|
||||
EnableSoftwareInventory bool `json:"enable_software_inventory"`
|
||||
EnableScheduledQueryStats bool `json:"enable_scheduled_query_stats"`
|
||||
AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"`
|
||||
EnableHostUsers bool `json:"enable_host_users"`
|
||||
EnableSoftwareInventory bool `json:"enable_software_inventory"`
|
||||
AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"`
|
||||
}
|
||||
|
||||
type OrderDirection int
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/server/config"
|
||||
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
"github.com/go-kit/kit/log"
|
||||
@ -365,6 +366,14 @@ FROM homebrew_packages;
|
||||
DirectIngestFunc: directIngestSoftware,
|
||||
}
|
||||
|
||||
var scheduledQueryStats = DetailQuery{
|
||||
Query: `
|
||||
SELECT *,
|
||||
(SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter
|
||||
FROM osquery_schedule`,
|
||||
DirectIngestFunc: directIngestScheduledQueryStats,
|
||||
}
|
||||
|
||||
var softwareLinux = DetailQuery{
|
||||
Query: `
|
||||
WITH cached_users AS (SELECT * FROM users)
|
||||
@ -491,14 +500,6 @@ FROM python_packages;
|
||||
DirectIngestFunc: directIngestSoftware,
|
||||
}
|
||||
|
||||
var scheduledQueryStats = DetailQuery{
|
||||
Query: `
|
||||
SELECT *,
|
||||
(SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter
|
||||
FROM osquery_schedule`,
|
||||
DirectIngestFunc: directIngestScheduledQueryStats,
|
||||
}
|
||||
|
||||
var usersQuery = DetailQuery{
|
||||
// Note we use the cached_groups CTE (`WITH` clause) here to suggest to SQLite that it generate
|
||||
// the `groups` table only once. Without doing this, on some Windows systems (Domain Controllers)
|
||||
@ -738,7 +739,7 @@ func directIngestMunkiInfo(ctx context.Context, logger log.Logger, host *fleet.H
|
||||
return ds.SetOrUpdateMunkiVersion(ctx, host.ID, rows[0]["version"])
|
||||
}
|
||||
|
||||
func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery {
|
||||
func GetDetailQueries(ac *fleet.AppConfig, fleetConfig config.FleetConfig) map[string]DetailQuery {
|
||||
generatedMap := make(map[string]DetailQuery)
|
||||
for key, query := range detailQueries {
|
||||
generatedMap[key] = query
|
||||
@ -754,7 +755,7 @@ func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery {
|
||||
generatedMap["users"] = usersQuery
|
||||
}
|
||||
|
||||
if ac != nil && ac.HostSettings.EnableScheduledQueryStats {
|
||||
if fleetConfig.App.EnableScheduledQueryStats {
|
||||
generatedMap["scheduled_query_stats"] = scheduledQueryStats
|
||||
}
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/server/config"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
"github.com/fleetdm/fleet/v4/server/mock"
|
||||
"github.com/go-kit/kit/log"
|
||||
@ -19,7 +20,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) {
|
||||
var initialHost fleet.Host
|
||||
host := initialHost
|
||||
|
||||
ingest := GetDetailQueries(nil)["network_interface"].IngestFunc
|
||||
ingest := GetDetailQueries(nil, config.FleetConfig{})["network_interface"].IngestFunc
|
||||
|
||||
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
|
||||
assert.Equal(t, initialHost, host)
|
||||
@ -110,7 +111,7 @@ func TestDetailQueryScheduledQueryStats(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
|
||||
ingest := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableScheduledQueryStats: true}})["scheduled_query_stats"].DirectIngestFunc
|
||||
ingest := GetDetailQueries(nil, config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}})["scheduled_query_stats"].DirectIngestFunc
|
||||
|
||||
ctx := context.Background()
|
||||
assert.NoError(t, ingest(ctx, log.NewNopLogger(), &host, ds, nil, false))
|
||||
@ -288,7 +289,7 @@ func sortedKeysCompare(t *testing.T, m map[string]DetailQuery, expectedKeys []st
|
||||
}
|
||||
|
||||
func TestGetDetailQueries(t *testing.T) {
|
||||
queriesNoConfig := GetDetailQueries(nil)
|
||||
queriesNoConfig := GetDetailQueries(nil, config.FleetConfig{})
|
||||
require.Len(t, queriesNoConfig, 11)
|
||||
baseQueries := []string{
|
||||
"network_interface",
|
||||
@ -305,11 +306,11 @@ func TestGetDetailQueries(t *testing.T) {
|
||||
}
|
||||
sortedKeysCompare(t, queriesNoConfig, baseQueries)
|
||||
|
||||
queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableScheduledQueryStats: true}})
|
||||
queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}})
|
||||
require.Len(t, queriesWithUsers, 13)
|
||||
sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users", "scheduled_query_stats"))
|
||||
|
||||
queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true, EnableScheduledQueryStats: true}})
|
||||
queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true}}, config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}})
|
||||
require.Len(t, queriesWithUsersAndSoftware, 16)
|
||||
sortedKeysCompare(t, queriesWithUsersAndSoftware,
|
||||
append(baseQueries, "users", "software_macos", "software_linux", "software_windows", "scheduled_query_stats"))
|
||||
|
||||
@ -129,7 +129,7 @@ func (svc Service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier
|
||||
}
|
||||
|
||||
// Save enrollment details if provided
|
||||
detailQueries := osquery_utils.GetDetailQueries(appConfig)
|
||||
detailQueries := osquery_utils.GetDetailQueries(appConfig, svc.config)
|
||||
save := false
|
||||
if r, ok := hostDetails["os_version"]; ok {
|
||||
err := detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r})
|
||||
@ -423,7 +423,7 @@ func (svc *Service) detailQueriesForHost(ctx context.Context, host *fleet.Host)
|
||||
}
|
||||
|
||||
queries := make(map[string]string)
|
||||
detailQueries := osquery_utils.GetDetailQueries(config)
|
||||
detailQueries := osquery_utils.GetDetailQueries(config, svc.config)
|
||||
for name, query := range detailQueries {
|
||||
if query.RunsForPlatform(host.Platform) {
|
||||
queries[hostDetailQueryPrefix+name] = query.Query
|
||||
@ -628,7 +628,7 @@ func (svc *Service) ingestDetailQuery(ctx context.Context, host *fleet.Host, nam
|
||||
return osqueryError{message: "ingest detail query: " + err.Error()}
|
||||
}
|
||||
|
||||
detailQueries := osquery_utils.GetDetailQueries(config)
|
||||
detailQueries := osquery_utils.GetDetailQueries(config, svc.config)
|
||||
query, ok := detailQueries[name]
|
||||
if !ok {
|
||||
return osqueryError{message: "unknown detail query " + name}
|
||||
@ -744,7 +744,7 @@ func (svc *Service) directIngestDetailQuery(ctx context.Context, host *fleet.Hos
|
||||
return false, osqueryError{message: "ingest detail query: " + err.Error()}
|
||||
}
|
||||
|
||||
detailQueries := osquery_utils.GetDetailQueries(config)
|
||||
detailQueries := osquery_utils.GetDetailQueries(config, svc.config)
|
||||
query, ok := detailQueries[name]
|
||||
if !ok {
|
||||
return false, osqueryError{message: "unknown detail query " + name}
|
||||
|
||||
@ -40,8 +40,7 @@ import (
|
||||
)
|
||||
|
||||
// One of these queries is the disk space, only one of the two works in a platform
|
||||
var expectedDetailQueries = len(osquery_utils.GetDetailQueries(
|
||||
&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}})) - 1
|
||||
var expectedDetailQueries = len(osquery_utils.GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, config.FleetConfig{})) - 1
|
||||
|
||||
func TestEnrollAgent(t *testing.T) {
|
||||
ds := new(mock.Store)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user