diff --git a/changes/add-enable-scheduled-query-stats-to-fleetconfig b/changes/add-enable-scheduled-query-stats-to-fleetconfig new file mode 100644 index 000000000..af78c58aa --- /dev/null +++ b/changes/add-enable-scheduled-query-stats-to-fleetconfig @@ -0,0 +1 @@ +* Allow to disable scheduled query stats through fleet serve config diff --git a/changes/allow-disabling-scheduled-query-stats b/changes/allow-disabling-scheduled-query-stats deleted file mode 100644 index d5db86b70..000000000 --- a/changes/allow-disabling-scheduled-query-stats +++ /dev/null @@ -1 +0,0 @@ -* Allow disabling scheduled query stats via app config diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index b8466ff64..0ac631f44 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -398,7 +398,6 @@ spec: host_expiry_window: 0 host_settings: enable_host_users: true - enable_scheduled_query_stats: false enable_software_inventory: false org_info: org_logo_url: "" @@ -450,7 +449,7 @@ spec: enable_vulnerabilities_webhook: false host_batch_size: 0 ` - expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}} + expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}} ` assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config"})) @@ -468,7 +467,6 @@ spec: host_expiry_window: 0 host_settings: enable_host_users: true - enable_scheduled_query_stats: false enable_software_inventory: false license: expiration: "0001-01-01T00:00:00Z" @@ -550,7 +548,7 @@ spec: enable_vulnerabilities_webhook: false host_batch_size: 0 ` - expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}} + expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}} ` assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config", "--include-server-config"})) diff --git a/docs/01-Using-Fleet/configuration-files/README.md b/docs/01-Using-Fleet/configuration-files/README.md index 3d28273dd..380c659e7 100644 --- a/docs/01-Using-Fleet/configuration-files/README.md +++ b/docs/01-Using-Fleet/configuration-files/README.md @@ -6,7 +6,6 @@ - [Enroll secrets](#enroll-secrets) - [Teams](#teams) - [Organization settings](#organization-settings) -- [Host settings](#host-settings) Entities in Fleet, such as queries, packs, labels, agent options, and enroll secrets, can be managed with configuration files in yaml syntax. @@ -495,9 +494,8 @@ in a public channel or a Github issue. ### Host settings -The `host_settings` section of the configuration yaml allows to define what predefined queries are sent to the hosts and +The `host_settings` section of the configuration yaml allows to define what predefined queries are sent to the hosts and later on processed by Fleet for different functionalities. - `host_settings.enable_host_users`: boolean value that when enabled Fleet will send the query needed to gather user data -- `host_settings.enable_software_inventory`: boolean value that when enabled Fleet will send the query needed to gather the list of software installed along with other metadata -- `host_settings.enable_scheduled_query_stats`: boolean value that when enabled Fleet will send the query needed to gather statistics about query executions in a host +- `host_settings.enable_software_inventory`: boolean value that when enabled Fleet will send the query needed to gather the list of software installed along with other metadata \ No newline at end of file diff --git a/docs/02-Deploying/03-Configuration.md b/docs/02-Deploying/03-Configuration.md index 6a3888681..d291bc729 100644 --- a/docs/02-Deploying/03-Configuration.md +++ b/docs/02-Deploying/03-Configuration.md @@ -769,6 +769,19 @@ How long invite tokens should be valid for. invite_token_validity_period: 1d ``` +##### app_enable_scheduled_query_stats + +Determines whether Fleet gets scheduled query statistics from hosts or not. + +- Default value: `true` +- Environment variable: `FLEET_APP_ENABLE_SCHEDULED_QUERY_STATS` +- Config file format: + + ``` + app: + enable_scheduled_query_stats: true + ``` + #### License ##### license_key diff --git a/server/config/config.go b/server/config/config.go index 888224673..cc9074373 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -93,6 +93,7 @@ type AuthConfig struct { type AppConfig struct { TokenKeySize int `yaml:"token_key_size"` InviteTokenValidityPeriod time.Duration `yaml:"invite_token_validity_period"` + EnableScheduledQueryStats bool `yaml:"enable_scheduled_query_stats"` } // SessionConfig defines configs related to user sessions @@ -385,6 +386,8 @@ func (man Manager) addConfigs() { "Duration invite tokens remain valid (i.e. 1h)") man.addConfigInt("app.token_key_size", 24, "Size of generated tokens") + man.addConfigBool("app.enable_scheduled_query_stats", true, + "If true (default) it gets scheduled query stats from hosts") // Session man.addConfigInt("session.key_size", 64, @@ -610,6 +613,7 @@ func (man Manager) LoadConfig() FleetConfig { App: AppConfig{ TokenKeySize: man.getConfigInt("app.token_key_size"), InviteTokenValidityPeriod: man.getConfigDuration("app.invite_token_validity_period"), + EnableScheduledQueryStats: man.getConfigBool("app.enable_scheduled_query_stats"), }, Session: SessionConfig{ KeySize: man.getConfigInt("session.key_size"), diff --git a/server/datastore/mysql/app_configs_test.go b/server/datastore/mysql/app_configs_test.go index d1d73a3fb..8959912da 100644 --- a/server/datastore/mysql/app_configs_test.go +++ b/server/datastore/mysql/app_configs_test.go @@ -280,7 +280,6 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) { require.Equal(t, 24*time.Hour, ac.WebhookSettings.Interval.Duration) require.False(t, ac.WebhookSettings.HostStatusWebhook.Enable) require.True(t, ac.HostSettings.EnableHostUsers) - require.True(t, ac.HostSettings.EnableScheduledQueryStats) require.False(t, ac.HostSettings.EnableSoftwareInventory) _, err = ds.writer.Exec( @@ -294,6 +293,5 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) { require.Equal(t, 12*time.Hour, ac.WebhookSettings.Interval.Duration) require.False(t, ac.HostSettings.EnableHostUsers) - require.True(t, ac.HostSettings.EnableScheduledQueryStats) require.False(t, ac.HostSettings.EnableSoftwareInventory) } diff --git a/server/fleet/app.go b/server/fleet/app.go index f81e331e7..527c7213c 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -230,7 +230,6 @@ func (c *AppConfig) ApplyDefaultsForNewInstalls() { func (c *AppConfig) ApplyDefaults() { c.HostSettings.EnableHostUsers = true - c.HostSettings.EnableScheduledQueryStats = true c.WebhookSettings.Interval.Duration = 24 * time.Hour } @@ -256,10 +255,9 @@ type HostExpirySettings struct { } type HostSettings struct { - EnableHostUsers bool `json:"enable_host_users"` - EnableSoftwareInventory bool `json:"enable_software_inventory"` - EnableScheduledQueryStats bool `json:"enable_scheduled_query_stats"` - AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"` + EnableHostUsers bool `json:"enable_host_users"` + EnableSoftwareInventory bool `json:"enable_software_inventory"` + AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"` } type OrderDirection int diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index d5ab0d616..1e98afadc 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/go-kit/kit/log" @@ -365,6 +366,14 @@ FROM homebrew_packages; DirectIngestFunc: directIngestSoftware, } +var scheduledQueryStats = DetailQuery{ + Query: ` + SELECT *, + (SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter + FROM osquery_schedule`, + DirectIngestFunc: directIngestScheduledQueryStats, +} + var softwareLinux = DetailQuery{ Query: ` WITH cached_users AS (SELECT * FROM users) @@ -491,14 +500,6 @@ FROM python_packages; DirectIngestFunc: directIngestSoftware, } -var scheduledQueryStats = DetailQuery{ - Query: ` - SELECT *, - (SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter - FROM osquery_schedule`, - DirectIngestFunc: directIngestScheduledQueryStats, -} - var usersQuery = DetailQuery{ // Note we use the cached_groups CTE (`WITH` clause) here to suggest to SQLite that it generate // the `groups` table only once. Without doing this, on some Windows systems (Domain Controllers) @@ -738,7 +739,7 @@ func directIngestMunkiInfo(ctx context.Context, logger log.Logger, host *fleet.H return ds.SetOrUpdateMunkiVersion(ctx, host.ID, rows[0]["version"]) } -func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery { +func GetDetailQueries(ac *fleet.AppConfig, fleetConfig config.FleetConfig) map[string]DetailQuery { generatedMap := make(map[string]DetailQuery) for key, query := range detailQueries { generatedMap[key] = query @@ -754,7 +755,7 @@ func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery { generatedMap["users"] = usersQuery } - if ac != nil && ac.HostSettings.EnableScheduledQueryStats { + if fleetConfig.App.EnableScheduledQueryStats { generatedMap["scheduled_query_stats"] = scheduledQueryStats } diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go index 13c823b72..43b8a7384 100644 --- a/server/service/osquery_utils/queries_test.go +++ b/server/service/osquery_utils/queries_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" "github.com/go-kit/kit/log" @@ -19,7 +20,7 @@ func TestDetailQueryNetworkInterfaces(t *testing.T) { var initialHost fleet.Host host := initialHost - ingest := GetDetailQueries(nil)["network_interface"].IngestFunc + ingest := GetDetailQueries(nil, config.FleetConfig{})["network_interface"].IngestFunc assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) assert.Equal(t, initialHost, host) @@ -110,7 +111,7 @@ func TestDetailQueryScheduledQueryStats(t *testing.T) { return nil } - ingest := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableScheduledQueryStats: true}})["scheduled_query_stats"].DirectIngestFunc + ingest := GetDetailQueries(nil, config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}})["scheduled_query_stats"].DirectIngestFunc ctx := context.Background() assert.NoError(t, ingest(ctx, log.NewNopLogger(), &host, ds, nil, false)) @@ -288,7 +289,7 @@ func sortedKeysCompare(t *testing.T, m map[string]DetailQuery, expectedKeys []st } func TestGetDetailQueries(t *testing.T) { - queriesNoConfig := GetDetailQueries(nil) + queriesNoConfig := GetDetailQueries(nil, config.FleetConfig{}) require.Len(t, queriesNoConfig, 11) baseQueries := []string{ "network_interface", @@ -305,11 +306,11 @@ func TestGetDetailQueries(t *testing.T) { } sortedKeysCompare(t, queriesNoConfig, baseQueries) - queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableScheduledQueryStats: true}}) + queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}}) require.Len(t, queriesWithUsers, 13) sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users", "scheduled_query_stats")) - queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true, EnableScheduledQueryStats: true}}) + queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true}}, config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}}) require.Len(t, queriesWithUsersAndSoftware, 16) sortedKeysCompare(t, queriesWithUsersAndSoftware, append(baseQueries, "users", "software_macos", "software_linux", "software_windows", "scheduled_query_stats")) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index c2d739ed6..397d383cf 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -129,7 +129,7 @@ func (svc Service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier } // Save enrollment details if provided - detailQueries := osquery_utils.GetDetailQueries(appConfig) + detailQueries := osquery_utils.GetDetailQueries(appConfig, svc.config) save := false if r, ok := hostDetails["os_version"]; ok { err := detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r}) @@ -423,7 +423,7 @@ func (svc *Service) detailQueriesForHost(ctx context.Context, host *fleet.Host) } queries := make(map[string]string) - detailQueries := osquery_utils.GetDetailQueries(config) + detailQueries := osquery_utils.GetDetailQueries(config, svc.config) for name, query := range detailQueries { if query.RunsForPlatform(host.Platform) { queries[hostDetailQueryPrefix+name] = query.Query @@ -628,7 +628,7 @@ func (svc *Service) ingestDetailQuery(ctx context.Context, host *fleet.Host, nam return osqueryError{message: "ingest detail query: " + err.Error()} } - detailQueries := osquery_utils.GetDetailQueries(config) + detailQueries := osquery_utils.GetDetailQueries(config, svc.config) query, ok := detailQueries[name] if !ok { return osqueryError{message: "unknown detail query " + name} @@ -744,7 +744,7 @@ func (svc *Service) directIngestDetailQuery(ctx context.Context, host *fleet.Hos return false, osqueryError{message: "ingest detail query: " + err.Error()} } - detailQueries := osquery_utils.GetDetailQueries(config) + detailQueries := osquery_utils.GetDetailQueries(config, svc.config) query, ok := detailQueries[name] if !ok { return false, osqueryError{message: "unknown detail query " + name} diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index 03a43d21c..c58eb1345 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -40,8 +40,7 @@ import ( ) // One of these queries is the disk space, only one of the two works in a platform -var expectedDetailQueries = len(osquery_utils.GetDetailQueries( - &fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}})) - 1 +var expectedDetailQueries = len(osquery_utils.GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}, config.FleetConfig{})) - 1 func TestEnrollAgent(t *testing.T) { ds := new(mock.Store)