diff --git a/changes/issue-3086-add-hosts-count-to-software b/changes/issue-3086-add-hosts-count-to-software new file mode 100644 index 000000000..0df2b9f31 --- /dev/null +++ b/changes/issue-3086-add-hosts-count-to-software @@ -0,0 +1 @@ +* Add `hosts_count` field for each software (and `counts_updated_at` timestamp as a top-level field) to the response payload of `GET /api/v1/fleet/software` diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index dcc518f90..c44da69b8 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -643,10 +643,12 @@ func cronVulnerabilities( level.Error(logger).Log("config", "couldn't read app config", "err", err) return } + + vulnDisabled := false if appConfig.VulnerabilitySettings.DatabasesPath == "" && config.Vulnerabilities.DatabasesPath == "" { level.Info(logger).Log("vulnerability scanning", "not configured") - return + vulnDisabled = true } if !appConfig.HostSettings.EnableSoftwareInventory { level.Info(logger).Log("software inventory", "not configured") @@ -664,15 +666,19 @@ func cronVulnerabilities( "result", vulnPath) } - level.Info(logger).Log("databases-path", vulnPath) + if !vulnDisabled { + level.Info(logger).Log("databases-path", vulnPath) + } level.Info(logger).Log("periodicity", config.Vulnerabilities.Periodicity) - if config.Vulnerabilities.CurrentInstanceChecks == "auto" { - level.Debug(logger).Log("current instance checks", "auto", "trying to create databases-path", vulnPath) - err := os.MkdirAll(vulnPath, 0o755) - if err != nil { - level.Error(logger).Log("databases-path", "creation failed, returning", "err", err) - return + if !vulnDisabled { + if config.Vulnerabilities.CurrentInstanceChecks == "auto" { + level.Debug(logger).Log("current instance checks", "auto", "trying to create databases-path", vulnPath) + err := os.MkdirAll(vulnPath, 0o755) + if err != nil { + level.Error(logger).Log("databases-path", "creation failed, returning", "err", err) + return + } } } @@ -694,16 +700,24 @@ func cronVulnerabilities( } } - err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config) - if err != nil { - level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err) - sentry.CaptureException(err) - continue + if !vulnDisabled { + err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config) + if err != nil { + level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err) + sentry.CaptureException(err) + continue + } + + err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config) + if err != nil { + level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err) + sentry.CaptureException(err) + continue + } } - err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config) - if err != nil { - level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err) + if err := ds.CalculateHostsPerSoftware(ctx, time.Now()); err != nil { + level.Error(logger).Log("msg", "calculating hosts count per software", "err", err) sentry.CaptureException(err) continue } diff --git a/docs/01-Using-Fleet/03-REST-API.md b/docs/01-Using-Fleet/03-REST-API.md index 543e2516c..cfdd89522 100644 --- a/docs/01-Using-Fleet/03-REST-API.md +++ b/docs/01-Using-Fleet/03-REST-API.md @@ -1063,7 +1063,7 @@ Request (`filters` is specified): Requires the [macadmins osquery extension](https://github.com/macadmins/osquery-extension) which comes bundled in [Fleet's osquery -installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer). +installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer). Retrieves a host's Google Chrome profile information which can be used to link a host to a specific user by email. @@ -1098,11 +1098,11 @@ user by email. --- -### Get host's mobile device management (MDM) and Munki information +### Get host's mobile device management (MDM) and Munki information Requires the [macadmins osquery extension](https://github.com/macadmins/osquery-extension) which comes bundled in [Fleet's osquery -installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer). +installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer). Retrieves a host's MDM enrollment status, MDM server URL, and Munki version. @@ -5872,10 +5872,10 @@ Transforms a host name into a host id. For example, the Fleet UI use this endpoi | ----------------------- | ------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | page | integer | query | Page number of the results to fetch. | | per_page | integer | query | Results per page. | -| order_key | string | query | What to order results by. Can be ordered by the following fields: `name`. | -| order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default is `asc`. | -| query | string | query | Search query keywords. Searchable fields include `name`. | -| team_id | integer | query | _Available in Fleet Premium_ Filters the software to only include the software installed on the hosts that are assigned to the specified team. | +| order_key | string | query | What to order results by. Can be ordered by the following fields: `name`, `hosts_count`. Defaults to the hosts count, descending. | +| order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default if not provided is `asc`. | +| query | string | query | Search query keywords. Searchable fields include `name`. | +| team_id | integer | query | _Available in Fleet Premium_ Filters the software to only include the software installed on the hosts that are assigned to the specified team. | | vulnerable | bool | query | If true or 1, only list software that has detected vulnerabilities | #### Example @@ -5888,22 +5888,16 @@ Transforms a host name into a host id. For example, the Fleet UI use this endpoi ```json { + "counts_updated_at": "2022-01-01 12:32:00", "software": [ { - "id": 1, - "name": "Chrome.app", + "id": 4, + "name": "osquery", "version": "2.1.11", - "source": "Application (macOS)", + "source": "rpm_packages", "generated_cpe": "", - "vulnerabilities": null - }, - { - "id": 2, - "name": "Figma.app", - "version": "2.1.11", - "source": "Application (macOS)", - "generated_cpe": "", - "vulnerabilities": null + "vulnerabilities": null, + "hosts_count": 456 }, { "id": 3, @@ -5911,15 +5905,26 @@ Transforms a host name into a host id. For example, the Fleet UI use this endpoi "version": "2.1.11", "source": "rpm_packages", "generated_cpe": "", - "vulnerabilities": null + "vulnerabilities": null, + "hosts_count": 345 }, { - "id": 4, - "name": "osquery", + "id": 2, + "name": "Figma.app", "version": "2.1.11", - "source": "rpm_packages", + "source": "Application (macOS)", "generated_cpe": "", - "vulnerabilities": null + "vulnerabilities": null, + "hosts_count": 234 + }, + { + "id": 1, + "name": "Chrome.app", + "version": "2.1.11", + "source": "Application (macOS)", + "generated_cpe": "", + "vulnerabilities": null, + "hosts_count": 123 } ] } diff --git a/docs/02-Deploying/03-Configuration.md b/docs/02-Deploying/03-Configuration.md index 6e9af6b93..7efdacc5b 100644 --- a/docs/02-Deploying/03-Configuration.md +++ b/docs/02-Deploying/03-Configuration.md @@ -1846,7 +1846,7 @@ When `current_instance_checks` is set to `auto` (the default), Fleet instances w ##### periodicity -How often vulnerabilities are checked. +How often vulnerabilities are checked. This is also the interval at which the counts of hosts per software is calculated. - Default value: `1h` - Environment variable: `FLEET_VULNERABILITIES_PERIODICITY` diff --git a/go.sum b/go.sum index b2b864220..e56e0448e 100644 --- a/go.sum +++ b/go.sum @@ -1051,6 +1051,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY= diff --git a/server/datastore/mysql/migrations/tables/20220125105650_UpdateAggregatedStats.go b/server/datastore/mysql/migrations/tables/20220125105650_UpdateAggregatedStats.go new file mode 100644 index 000000000..d48a97a56 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220125105650_UpdateAggregatedStats.go @@ -0,0 +1,25 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20220125105650, Down_20220125105650) +} + +func Up_20220125105650(tx *sql.Tx) error { + if _, err := tx.Exec(`ALTER TABLE aggregated_stats MODIFY id bigint(20) unsigned NOT NULL`); err != nil { + return errors.Wrap(err, "make aggregated_stats.id bigint") + } + if _, err := tx.Exec("create index aggregated_stats_type_idx on aggregated_stats(`type`);"); err != nil { + return errors.Wrap(err, "creating aggregated_stats index") + } + return nil +} + +func Down_20220125105650(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 0933ba267..6643cc207 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -15,13 +15,14 @@ CREATE TABLE `activities` ( /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `aggregated_stats` ( - `id` int(10) unsigned NOT NULL, + `id` bigint(20) unsigned NOT NULL, `type` varchar(255) NOT NULL, `json_value` json NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`,`type`), - KEY `idx_aggregated_stats_updated_at` (`updated_at`) + KEY `idx_aggregated_stats_updated_at` (`updated_at`), + KEY `aggregated_stats_type_idx` (`type`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; @@ -316,9 +317,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=119 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=120 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index a461d7617..d7cb162ce 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/doug-martin/goqu/v9" _ "github.com/doug-martin/goqu/v9/dialect/mysql" @@ -306,6 +307,19 @@ func selectSoftwareSQL(hostID *uint, opts fleet.SoftwareListOptions) (string, [] ) } + if opts.WithHostCounts { + ds = ds.Join( + goqu.I("aggregated_stats").As("shc"), + goqu.On( + goqu.I("s.id").Eq(goqu.I("shc.id")), + ), + ).Where(goqu.I("shc.type").Eq("software_hosts_count"), goqu.I("shc.json_value").Gt(0)). + SelectAppend( + goqu.I("shc.json_value").As("hosts_count"), + goqu.I("shc.updated_at").As("counts_updated_at"), + ) + } + return ds.ToSQL() } @@ -519,3 +533,87 @@ func (d *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software, return &software, nil } + +// CalculateHostsPerSoftware calculates the number of hosts having each +// software installed and stores that information in the aggregated_stats +// table. +func (d *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error { + // NOTE(mna): for reference, on my laptop I get ~1.5ms for 10_000 hosts / 100 software each, + // ~1.5s for 10_000 hosts / 1_000 software each (but this is with an otherwise empty + // aggregated_stats table, but still reasonable numbers give that this runs as a cron + // task in the background). + + resetStmt := ` + UPDATE aggregated_stats + SET json_value = CAST(0 AS json) + WHERE type = "software_hosts_count"` + + queryStmt := ` + SELECT count(*), software_id + FROM host_software + GROUP BY software_id` + + insertStmt := ` + INSERT INTO aggregated_stats + (id, type, json_value, updated_at) + VALUES + %s + ON DUPLICATE KEY UPDATE + json_value = VALUES(json_value), + updated_at = VALUES(updated_at)` + valuesPart := `(?, "software_hosts_count", CAST(? AS json), ?),` + + // first, reset all counts to 0 + if _, err := d.writer.ExecContext(ctx, resetStmt); err != nil { + return ctxerr.Wrap(ctx, err, "reset all software_hosts_count to 0 in aggregated_stats") + } + + // next get a cursor for the counts for each software + rows, err := d.reader.QueryContext(ctx, queryStmt) + if err != nil { + return ctxerr.Wrap(ctx, err, "read counts from host_software") + } + defer rows.Close() + + // use a loop to iterate to prevent loading all in one go in memory, as it + // could get pretty big at >100K hosts with 1000+ software each. + const batchSize = 100 + var batchCount int + args := make([]interface{}, 0, batchSize*3) + for rows.Next() { + var count int + var sid uint + + if err := rows.Scan(&count, &sid); err != nil { + return ctxerr.Wrap(ctx, err, "scan row into variables") + } + + args = append(args, sid, count, updatedAt) + batchCount++ + + if batchCount == batchSize { + values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") + if _, err := d.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert batch into aggregated_stats") + } + + args = args[:0] + batchCount = 0 + } + } + if batchCount > 0 { + values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") + if _, err := d.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert batch into aggregated_stats") + } + } + if err := rows.Err(); err != nil { + return ctxerr.Wrap(ctx, err, "iterate over host_software counts") + } + + // TODO(mna): delete any unused software from the software table (any that + // isn't in that list with a host count > 0). This also addresses another + // TODO in this file. + + return nil +} diff --git a/server/datastore/mysql/software_bench_test.go b/server/datastore/mysql/software_bench_test.go new file mode 100644 index 000000000..be05ebd42 --- /dev/null +++ b/server/datastore/mysql/software_bench_test.go @@ -0,0 +1,170 @@ +package mysql + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server" + "github.com/stretchr/testify/require" +) + +func BenchmarkCalculateHostsPerSoftware(b *testing.B) { + ts := time.Now() + type counts struct{ hs, sws int } + + cases := []counts{ + {1, 1}, + {10, 10}, + {100, 100}, + {1_000, 100}, + {10_000, 100}, + {10_000, 1_000}, + } + + b.Run("resetUpdate", func(b *testing.B) { + b.Run("singleSelectGroupByInsertBatch100AggStats", func(b *testing.B) { + for _, c := range cases { + b.Run(fmt.Sprintf("%d:%d", c.hs, c.sws), func(b *testing.B) { + ds := CreateMySQLDS(b) + generateHostsWithSoftware(b, ds, c.hs, c.sws) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + resetUpdateAllZeroAgg(b, ds) + singleSelectGroupByInsertBatchAgg(b, ds, ts, 100) + } + checkCountsAgg(b, ds, c.hs, c.sws) + }) + } + }) + }) +} + +func checkCountsAgg(b *testing.B, ds *Datastore, hs, sws int) { + var rowsCount, invalidHostsCount int + + rowsStmt := `SELECT COUNT(*) FROM aggregated_stats WHERE type = "software_hosts_count"` + err := ds.writer.GetContext(context.Background(), &rowsCount, rowsStmt) + require.NoError(b, err) + require.Equal(b, sws, rowsCount) + + invalidStmt := `SELECT COUNT(*) FROM aggregated_stats WHERE type = "software_hosts_count" AND json_value != CAST(? AS json)` + err = ds.writer.GetContext(context.Background(), &invalidHostsCount, invalidStmt, hs) + require.NoError(b, err) + require.Equal(b, 0, invalidHostsCount) +} + +func generateHostsWithSoftware(b *testing.B, ds *Datastore, hs, sws int) { + hostInsert := ` + INSERT INTO hosts ( + osquery_host_id, + node_key, + hostname, + uuid + ) + VALUES ` + hostValuePart := `(?, ?, ?, ?),` + + var sb strings.Builder + sb.WriteString(hostInsert) + args := make([]interface{}, 0, hs*4) + for i := 0; i < hs; i++ { + osqueryHostID, _ := server.GenerateRandomText(10) + name := "host" + strconv.Itoa(i) + args = append(args, osqueryHostID, name+"key", name, name+"uuid") + sb.WriteString(hostValuePart) + } + stmt := strings.TrimSuffix(sb.String(), ",") + _, err := ds.writer.ExecContext(context.Background(), stmt, args...) + require.NoError(b, err) + + swInsert := ` + INSERT INTO software ( + name, + version, + source + ) VALUES ` + swValuePart := `(?, ?, ?),` + + sb.Reset() + sb.WriteString(swInsert) + args = make([]interface{}, 0, sws*3) + for i := 0; i < sws; i++ { + name := "software" + strconv.Itoa(i) + args = append(args, name, strconv.Itoa(i)+".0.0", "testing") + sb.WriteString(swValuePart) + } + stmt = strings.TrimSuffix(sb.String(), ",") + _, err = ds.writer.ExecContext(context.Background(), stmt, args...) + require.NoError(b, err) + + // cartesian product of hosts and software tables + hostSwInsert := ` + INSERT INTO host_software (host_id, software_id) + SELECT + h.id, + sw.id + FROM + hosts h, + software sw` + _, err = ds.writer.ExecContext(context.Background(), hostSwInsert) + require.NoError(b, err) +} + +func resetUpdateAllZeroAgg(b *testing.B, ds *Datastore) { + updateStmt := `UPDATE aggregated_stats SET json_value = CAST(0 AS json) WHERE type = "software_hosts_count"` + _, err := ds.writer.ExecContext(context.Background(), updateStmt) + require.NoError(b, err) +} + +func singleSelectGroupByInsertBatchAgg(b *testing.B, ds *Datastore, updatedAt time.Time, batchSize int) { + queryStmt := ` + SELECT count(*), software_id + FROM host_software + GROUP BY software_id` + + insertStmt := ` + INSERT INTO aggregated_stats + (id, type, json_value, updated_at) + VALUES + %s + ON DUPLICATE KEY UPDATE + json_value = VALUES(json_value), + updated_at = VALUES(updated_at)` + valuesPart := `(?, "software_hosts_count", CAST(? AS json), ?),` + + rows, err := ds.reader.QueryContext(context.Background(), queryStmt) + require.NoError(b, err) + defer rows.Close() + + var batchCount int + args := make([]interface{}, 0, batchSize*3) + for rows.Next() { + var count int + var sid uint + + require.NoError(b, rows.Scan(&count, &sid)) + args = append(args, sid, count, updatedAt) + batchCount++ + + if batchCount == batchSize { + values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") + _, err := ds.writer.ExecContext(context.Background(), fmt.Sprintf(insertStmt, values), args...) + require.NoError(b, err) + + args = args[:0] + batchCount = 0 + } + } + + if batchCount > 0 { + values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") + _, err := ds.writer.ExecContext(context.Background(), fmt.Sprintf(insertStmt, values), args...) + require.NoError(b, err) + } + require.NoError(b, rows.Err()) +} diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index fc3aea2cd..3640562e4 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -401,19 +401,19 @@ func testSoftwareList(t *testing.T, ds *Datastore) { bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages"} t.Run("lists everything", func(t *testing.T) { - software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{}) + software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{}, true) expected := []fleet.Software{bar003, foo001, foo003, foo002} test.ElementsMatchSkipID(t, software, expected) }) t.Run("limits the results", func(t *testing.T) { - software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{PerPage: 1, OrderKey: "version"}}) + software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{PerPage: 1, OrderKey: "version"}}, true) expected := []fleet.Software{foo001} test.ElementsMatchSkipID(t, software, expected) }) t.Run("paginates", func(t *testing.T) { - software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}}) + software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}}, true) expected := []fleet.Software{foo003} test.ElementsMatchSkipID(t, software, expected) }) @@ -423,7 +423,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) { require.NoError(t, err) require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID})) - software := listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "version"}, TeamID: &team1.ID}) + software := listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "version"}, TeamID: &team1.ID}, true) expected := []fleet.Software{foo001, foo003} test.ElementsMatchSkipID(t, software, expected) }) @@ -440,34 +440,34 @@ func testSoftwareList(t *testing.T, ds *Datastore) { OrderKey: "id", }, TeamID: &team1.ID, - }) + }, true) expected := []fleet.Software{foo003} test.ElementsMatchSkipID(t, software, expected) }) t.Run("filters vulnerable software", func(t *testing.T) { - software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{VulnerableOnly: true}) + software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{VulnerableOnly: true}, true) expected := []fleet.Software{foo001} test.ElementsMatchSkipID(t, software, expected) }) t.Run("filters by query", func(t *testing.T) { // query by name (case insensitive) - software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "baR"}}) + software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "baR"}}, true) expected := []fleet.Software{bar003} test.ElementsMatchSkipID(t, software, expected) // query by version - software = listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "0.0.3"}}) + software = listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "0.0.3"}}, true) expected = []fleet.Software{foo003, bar003} test.ElementsMatchSkipID(t, software, expected) // query by version (case insensitive) - software = listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "V0.0.2"}}) + software = listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "V0.0.2"}}, true) expected = []fleet.Software{foo002} test.ElementsMatchSkipID(t, software, expected) }) t.Run("can order by name and id", func(t *testing.T) { - software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "name,id", OrderDirection: fleet.OrderAscending}}) + software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "name,id", OrderDirection: fleet.OrderAscending}}, false) assert.Equal(t, bar003.Name, software[0].Name) assert.Equal(t, bar003.Version, software[0].Version) assert.Equal(t, bar003.Source, software[0].Source) @@ -476,9 +476,21 @@ func testSoftwareList(t *testing.T, ds *Datastore) { assert.Greater(t, software[2].ID, software[1].ID) assert.Greater(t, software[3].ID, software[2].ID) }) + + t.Run("hosts count", func(t *testing.T) { + defer TruncateTables(t, ds, "aggregated_stats") + listSoftwareCheckCount(t, ds, 0, 0, fleet.SoftwareListOptions{WithHostCounts: true}, false) + + // create the counts for those software and re-run + require.NoError(t, ds.CalculateHostsPerSoftware(context.Background(), time.Now())) + software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, WithHostCounts: true}, false) + // ordered by counts descending, so foo003 is first + assert.Equal(t, foo003.Name, software[0].Name) + assert.Equal(t, 2, software[0].HostsCount) + }) } -func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareListOptions) []fleet.Software { +func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareListOptions, returnSorted bool) []fleet.Software { software, err := ds.ListSoftware(context.Background(), opts) require.NoError(t, err) require.Len(t, software, expectedListCount) @@ -490,8 +502,11 @@ func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int, return s.Vulnerabilities[i].CVE < s.Vulnerabilities[j].CVE }) } - sort.Slice(software, func(i, j int) bool { - return software[i].Name+software[i].Version < software[j].Name+software[j].Version - }) + + if returnSorted { + sort.Slice(software, func(i, j int) bool { + return software[i].Name+software[i].Version < software[j].Name+software[j].Version + }) + } return software } diff --git a/server/datastore/mysql/testing_utils.go b/server/datastore/mysql/testing_utils.go index 41743d782..88c931a67 100644 --- a/server/datastore/mysql/testing_utils.go +++ b/server/datastore/mysql/testing_utils.go @@ -243,6 +243,11 @@ func createMySQLDSWithOptions(t testing.TB, opts *DatastoreTestOptions) *Datasto strings.TrimPrefix(details.Name(), "github.com/fleetdm/fleet/v4/"), "/", "_", ) cleanName = strings.ReplaceAll(cleanName, ".", "_") + if len(cleanName) > 60 { + // the later parts are more unique than the start, with the package names, + // so trim from the start. + cleanName = cleanName[len(cleanName)-60:] + } ds := initializeDatabase(t, cleanName, opts) t.Cleanup(func() { ds.Close() }) return ds diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index e1773555f..39c5bbc2c 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -326,6 +326,7 @@ type Datastore interface { AllCPEs(ctx context.Context) ([]string, error) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error SoftwareByID(ctx context.Context, id uint) (*Software, error) + CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error /////////////////////////////////////////////////////////////////////////////// // ActivitiesStore diff --git a/server/fleet/software.go b/server/fleet/software.go index bdd235657..68ceb92da 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -1,5 +1,7 @@ package fleet +import "time" + type SoftwareCVE struct { CVE string `json:"cve" db:"cve"` DetailsLink string `json:"details_link" db:"details_link"` @@ -21,6 +23,12 @@ type Software struct { GenerateCPE string `json:"generated_cpe" db:"generated_cpe"` // Vulnerabilities lists all the found CVEs for the CPE Vulnerabilities VulnerabilitiesSlice `json:"vulnerabilities"` + // HostsCount indicates the number of hosts with that software, filled only + // if explicitly requested. + HostsCount int `json:"hosts_count,omitempty" db:"hosts_count"` + // CountsUpdatedAt is the timestamp when the hosts count was last updated + // for that software, filled only if hosts count is requested. + CountsUpdatedAt time.Time `json:"-" db:"counts_updated_at"` } func (Software) AuthzType() string { @@ -54,4 +62,9 @@ type SoftwareListOptions struct { VulnerableOnly bool `query:"vulnerable,optional"` SkipLoadingCVEs bool + + // WithHostCounts indicates that the list of software should include the + // counts of hosts per software, and include only those software that have + // a count of hosts > 0. + WithHostCounts bool } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 1aa43e4e6..ed63eda1d 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -266,6 +266,8 @@ type InsertCVEForCPEFunc func(ctx context.Context, cve string, cpes []string) er type SoftwareByIDFunc func(ctx context.Context, id uint) (*fleet.Software, error) +type CalculateHostsPerSoftwareFunc func(ctx context.Context, updatedAt time.Time) error + type NewActivityFunc func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error type ListActivitiesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Activity, error) @@ -742,6 +744,9 @@ type DataStore struct { SoftwareByIDFunc SoftwareByIDFunc SoftwareByIDFuncInvoked bool + CalculateHostsPerSoftwareFunc CalculateHostsPerSoftwareFunc + CalculateHostsPerSoftwareFuncInvoked bool + NewActivityFunc NewActivityFunc NewActivityFuncInvoked bool @@ -1519,6 +1524,11 @@ func (s *DataStore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software, return s.SoftwareByIDFunc(ctx, id) } +func (s *DataStore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error { + s.CalculateHostsPerSoftwareFuncInvoked = true + return s.CalculateHostsPerSoftwareFunc(ctx, updatedAt) +} + func (s *DataStore) NewActivity(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { s.NewActivityFuncInvoked = true return s.NewActivityFunc(ctx, user, activityType, details) diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index b60c5df7f..6f41bde55 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -403,20 +403,35 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { s.DoJSON("GET", "/api/v1/fleet/software/count", countReq, http.StatusOK, &countResp) assert.Equal(t, 2, countResp.Count) - lsReq := listSoftwareRequest{} - lsResp := listSoftwareResponse{} - s.DoJSON("GET", "/api/v1/fleet/software", lsReq, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") - assert.Len(t, lsResp.Software, 1) - assert.Equal(t, soft1.ID, lsResp.Software[0].ID) - assert.Len(t, lsResp.Software[0].Vulnerabilities, 1) + // no software host counts have been calculated yet, so this returns nothing + var lsResp listSoftwareResponse + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") + require.Len(t, lsResp.Software, 0) + assert.True(t, lsResp.CountsUpdatedAt.IsZero()) + // the software/count endpoint is different, it doesn't care about hosts counts s.DoJSON("GET", "/api/v1/fleet/software/count", countReq, http.StatusOK, &countResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") assert.Equal(t, 1, countResp.Count) - s.DoJSON("GET", "/api/v1/fleet/software", lsReq, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "name,id", "order_direction", "desc") - assert.Len(t, lsResp.Software, 1) + // calculate hosts counts + hostsCountTs := time.Now().UTC() + require.NoError(t, s.ds.CalculateHostsPerSoftware(context.Background(), hostsCountTs)) + + // now the list software endpoint returns the software + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") + require.Len(t, lsResp.Software, 1) assert.Equal(t, soft1.ID, lsResp.Software[0].ID) assert.Len(t, lsResp.Software[0].Vulnerabilities, 1) + assert.WithinDuration(t, hostsCountTs, lsResp.CountsUpdatedAt, time.Second) + + // the count endpoint still returns 1 + s.DoJSON("GET", "/api/v1/fleet/software/count", countReq, http.StatusOK, &countResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc") + assert.Equal(t, 1, countResp.Count) + + // default sort, not only vulnerable + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp) + require.True(t, len(lsResp.Software) >= len(software)) + assert.WithinDuration(t, hostsCountTs, lsResp.CountsUpdatedAt, time.Second) } func (s *integrationTestSuite) TestGlobalPolicies() { diff --git a/server/service/software.go b/server/service/software.go index 88edbcfaf..4cf1cdba9 100644 --- a/server/service/software.go +++ b/server/service/software.go @@ -2,6 +2,7 @@ package service import ( "context" + "time" "github.com/fleetdm/fleet/v4/server/fleet" ) @@ -15,8 +16,9 @@ type listSoftwareRequest struct { } type listSoftwareResponse struct { - Software []fleet.Software `json:"software,omitempty"` - Err error `json:"error,omitempty"` + CountsUpdatedAt time.Time `json:"counts_updated_at,omitempty"` + Software []fleet.Software `json:"software,omitempty"` + Err error `json:"error,omitempty"` } func (r listSoftwareResponse) error() error { return r.Err } @@ -27,7 +29,14 @@ func listSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Se if err != nil { return listSoftwareResponse{Err: err}, nil } - return listSoftwareResponse{Software: resp}, nil + + var latest time.Time + for _, sw := range resp { + if !sw.CountsUpdatedAt.IsZero() && sw.CountsUpdatedAt.After(latest) { + latest = sw.CountsUpdatedAt + } + } + return listSoftwareResponse{CountsUpdatedAt: latest, Software: resp}, nil } func (svc Service) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) { @@ -35,6 +44,12 @@ func (svc Service) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptio return nil, err } + // default sort order to hosts_count descending + if opt.OrderKey == "" { + opt.OrderKey = "hosts_count" + opt.OrderDirection = fleet.OrderDescending + } + opt.WithHostCounts = true return svc.ds.ListSoftware(ctx, opt) } diff --git a/server/service/software_test.go b/server/service/software_test.go index f76a974dd..b70420518 100644 --- a/server/service/software_test.go +++ b/server/service/software_test.go @@ -34,5 +34,17 @@ func TestService_ListSoftware(t *testing.T) { assert.True(t, ds.ListSoftwareFuncInvoked) assert.Equal(t, ptr.Uint(42), calledWithTeamID) - assert.Equal(t, fleet.ListOptions{PerPage: 77, Page: 4}, calledWithOpt.ListOptions) + // sort order defaults to hosts_count descending, automatically, if not explicitly provided + assert.Equal(t, fleet.ListOptions{PerPage: 77, Page: 4, OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, calledWithOpt.ListOptions) + assert.True(t, calledWithOpt.WithHostCounts) + + // call again, this time with an explicit sort + ds.ListSoftwareFuncInvoked = false + _, err = svc.ListSoftware(ctx, fleet.SoftwareListOptions{TeamID: nil, ListOptions: fleet.ListOptions{PerPage: 11, Page: 2, OrderKey: "id", OrderDirection: fleet.OrderAscending}}) + require.NoError(t, err) + + assert.True(t, ds.ListSoftwareFuncInvoked) + assert.Nil(t, calledWithTeamID) + assert.Equal(t, fleet.ListOptions{PerPage: 11, Page: 2, OrderKey: "id", OrderDirection: fleet.OrderAscending}, calledWithOpt.ListOptions) + assert.True(t, calledWithOpt.WithHostCounts) } diff --git a/server/test/new_objects.go b/server/test/new_objects.go index 0057cc144..801e627dc 100644 --- a/server/test/new_objects.go +++ b/server/test/new_objects.go @@ -98,7 +98,7 @@ func AddAllHostsLabel(t *testing.T, ds fleet.Datastore) { require.NoError(t, err) } -func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now time.Time) *fleet.Host { +func NewHost(tb testing.TB, ds fleet.Datastore, name, ip, key, uuid string, now time.Time) *fleet.Host { osqueryHostID, _ := server.GenerateRandomText(10) h, err := ds.NewHost(context.Background(), &fleet.Host{ Hostname: name, @@ -112,9 +112,9 @@ func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now t Platform: "darwin", }) - require.NoError(t, err) - require.NotZero(t, h.ID) - require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, now)) + require.NoError(tb, err) + require.NotZero(tb, h.ID) + require.NoError(tb, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, now)) return h }