diff --git a/server/datastore/mysql/migrations/tables/20220208144830_AddSoftwareHostCountsTable.go b/server/datastore/mysql/migrations/tables/20220208144830_AddSoftwareHostCountsTable.go new file mode 100644 index 000000000..01f8af213 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220208144830_AddSoftwareHostCountsTable.go @@ -0,0 +1,34 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20220208144830, Down_20220208144830) +} + +func Up_20220208144830(tx *sql.Tx) error { + softwareHostCountsTable := ` + CREATE TABLE IF NOT EXISTS software_host_counts ( + software_id bigint(20) unsigned NOT NULL, + hosts_count int(10) unsigned NOT NULL, + created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + PRIMARY KEY (software_id), + INDEX idx_software_host_counts_host_count_software_id (hosts_count, software_id), + INDEX idx_software_host_counts_updated_at_software_id (updated_at, software_id) + ); + ` + if _, err := tx.Exec(softwareHostCountsTable); err != nil { + return errors.Wrap(err, "create software_host_counts table") + } + return nil +} + +func Down_20220208144830(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index f48880124..d817811d2 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -317,9 +317,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=121 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=122 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( @@ -553,6 +553,18 @@ CREATE TABLE `software_cve` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `software_host_counts` ( + `software_id` bigint(20) unsigned NOT NULL, + `hosts_count` int(10) unsigned NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`software_id`), + KEY `idx_software_host_counts_host_count_software_id` (`hosts_count`,`software_id`), + KEY `idx_software_host_counts_updated_at_software_id` (`updated_at`,`software_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `statistics` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 4f71066a3..eb98ce37f 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -222,7 +222,7 @@ func listSoftwareDB( var result []fleet.Software if err := sqlx.SelectContext(ctx, q, &result, sql, args...); err != nil { - return nil, ctxerr.Wrap(ctx, err, "load host software") + return nil, ctxerr.Wrap(ctx, err, "select host software") } if opts.SkipLoadingCVEs { @@ -311,38 +311,18 @@ func selectSoftwareSQL(hostID *uint, opts fleet.SoftwareListOptions) (string, [] ) } - topLevelListOpts := opts.ListOptions - if opts.WithHostCounts { - subSelectCounts := dialect.From(goqu.I("aggregated_stats").As("shc")).Select( - "shc.id", goqu.I("shc.json_value").As("hosts_count"), goqu.I("shc.updated_at").As("counts_updated_at"), - ).Where(goqu.I("shc.type").Eq("software_hosts_count"), goqu.I("shc.json_value").Gt(0)) - - subSelectListOpts := opts.ListOptions - switch subSelectListOpts.OrderKey { - case "hosts_count", "counts_updated_at": - // all good, known columns, so we sort - subSelectCounts = appendListOptionsToSelect(subSelectCounts, opts.ListOptions) - // since the aggregated_stats table will be properly LIMITed and OFFSET, then - // we must not LIMIT and OFFSET the top-level query again (it can't return - // more rows than this internal query, and it must not be offset as the sub-query - // is already offset, so offsetting the top-level query IN ADDITION would offset - // it past any result. - topLevelListOpts.Page, topLevelListOpts.PerPage = 0, 0 - default: - // we don't sort if it's not a column from this table - } ds = ds.Join( - subSelectCounts.As("shc"), - goqu.On( - goqu.I("s.id").Eq(goqu.I("shc.id")), - ), - ).SelectAppend( - goqu.I("shc.hosts_count"), - goqu.I("shc.counts_updated_at"), - ) + goqu.I("software_host_counts").As("shc"), + goqu.On(goqu.I("s.id").Eq(goqu.I("shc.software_id"))), + ). + Where(goqu.I("shc.hosts_count").Gt(0)). + SelectAppend( + goqu.I("shc.hosts_count"), + goqu.I("shc.updated_at").As("counts_updated_at"), + ) } - ds = appendListOptionsToSelect(ds, topLevelListOpts) + ds = appendListOptionsToSelect(ds, opts.ListOptions) return ds.ToSQL() } @@ -563,18 +543,12 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software } // CalculateHostsPerSoftware calculates the number of hosts having each -// software installed and stores that information in the aggregated_stats +// software installed and stores that information in the software_host_counts // table. func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error { - // NOTE(mna): for reference, on my laptop I get ~1.5ms for 10_000 hosts / 100 software each, - // ~1.5s for 10_000 hosts / 1_000 software each (but this is with an otherwise empty - // aggregated_stats table, but still reasonable numbers give that this runs as a cron - // task in the background). - resetStmt := ` - UPDATE aggregated_stats - SET json_value = CAST(0 AS json) - WHERE type = "software_hosts_count"` + UPDATE software_host_counts + SET hosts_count = 0, updated_at = ?` queryStmt := ` SELECT count(*), software_id @@ -583,18 +557,18 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti GROUP BY software_id` insertStmt := ` - INSERT INTO aggregated_stats - (id, type, json_value, updated_at) + INSERT INTO software_host_counts + (software_id, hosts_count, updated_at) VALUES %s ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), + hosts_count = VALUES(hosts_count), updated_at = VALUES(updated_at)` - valuesPart := `(?, "software_hosts_count", CAST(? AS json), ?),` + valuesPart := `(?, ?, ?),` // first, reset all counts to 0 - if _, err := ds.writer.ExecContext(ctx, resetStmt); err != nil { - return ctxerr.Wrap(ctx, err, "reset all software_hosts_count to 0 in aggregated_stats") + if _, err := ds.writer.ExecContext(ctx, resetStmt, updatedAt); err != nil { + return ctxerr.Wrap(ctx, err, "reset all software_host_counts to 0") } // next get a cursor for the counts for each software @@ -623,7 +597,7 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti if batchCount == batchSize { values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") if _, err := ds.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { - return ctxerr.Wrap(ctx, err, "insert batch into aggregated_stats") + return ctxerr.Wrap(ctx, err, "insert batch into software_host_counts") } args = args[:0] @@ -633,7 +607,7 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti if batchCount > 0 { values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") if _, err := ds.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil { - return ctxerr.Wrap(ctx, err, "insert batch into aggregated_stats") + return ctxerr.Wrap(ctx, err, "insert last batch into software_host_counts") } } if err := rows.Err(); err != nil { @@ -649,11 +623,10 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti NOT EXISTS ( SELECT 1 FROM - aggregated_stats shc + software_host_counts shc WHERE - software.id = shc.id AND - shc.type = "software_hosts_count" AND - json_value > 0)` + software.id = shc.software_id AND + shc.hosts_count > 0)` if _, err := ds.writer.ExecContext(ctx, cleanupStmt); err != nil { return ctxerr.Wrap(ctx, err, "delete unused software") } diff --git a/server/datastore/mysql/software_bench_test.go b/server/datastore/mysql/software_bench_test.go deleted file mode 100644 index 05113bcde..000000000 --- a/server/datastore/mysql/software_bench_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package mysql - -import ( - "context" - "fmt" - "strconv" - "strings" - "testing" - "time" - - "github.com/fleetdm/fleet/v4/server" - "github.com/stretchr/testify/require" -) - -func BenchmarkCalculateHostsPerSoftware(b *testing.B) { - ts := time.Now() - type counts struct{ hs, sws int } - - cases := []counts{ - {1, 1}, - {10, 10}, - {100, 100}, - {1_000, 100}, - {10_000, 100}, - {10_000, 1_000}, - } - - b.Run("resetUpdate", func(b *testing.B) { - b.Run("singleSelectGroupByInsertBatch100AggStats", func(b *testing.B) { - for _, c := range cases { - b.Run(fmt.Sprintf("%d:%d", c.hs, c.sws), func(b *testing.B) { - ds := CreateMySQLDS(b) - generateHostsWithSoftware(b, ds, c.hs, c.sws) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - resetUpdateAllZeroAgg(b, ds) - singleSelectGroupByInsertBatchAgg(b, ds, ts, 100) - } - checkCountsAgg(b, ds, c.hs, c.sws) - }) - } - }) - }) - b.Run("CalculateHostsPerSoftware", func(b *testing.B) { - for _, c := range cases { - b.Run(fmt.Sprintf("%d:%d", c.hs, c.sws), func(b *testing.B) { - ctx := context.Background() - ds := CreateMySQLDS(b) - generateHostsWithSoftware(b, ds, c.hs, c.sws) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - require.NoError(b, ds.CalculateHostsPerSoftware(ctx, ts)) - } - checkCountsAgg(b, ds, c.hs, c.sws) - }) - } - }) -} - -func checkCountsAgg(b *testing.B, ds *Datastore, hs, sws int) { - var rowsCount, invalidHostsCount int - - rowsStmt := `SELECT COUNT(*) FROM aggregated_stats WHERE type = "software_hosts_count"` - err := ds.writer.GetContext(context.Background(), &rowsCount, rowsStmt) - require.NoError(b, err) - require.Equal(b, sws, rowsCount) - - invalidStmt := `SELECT COUNT(*) FROM aggregated_stats WHERE type = "software_hosts_count" AND json_value != CAST(? AS json)` - err = ds.writer.GetContext(context.Background(), &invalidHostsCount, invalidStmt, hs) - require.NoError(b, err) - require.Equal(b, 0, invalidHostsCount) -} - -func generateHostsWithSoftware(b *testing.B, ds *Datastore, hs, sws int) { - hostInsert := ` - INSERT INTO hosts ( - osquery_host_id, - node_key, - hostname, - uuid - ) - VALUES ` - hostValuePart := `(?, ?, ?, ?),` - - var sb strings.Builder - sb.WriteString(hostInsert) - args := make([]interface{}, 0, hs*4) - for i := 0; i < hs; i++ { - osqueryHostID, _ := server.GenerateRandomText(10) - name := "host" + strconv.Itoa(i) - args = append(args, osqueryHostID, name+"key", name, name+"uuid") - sb.WriteString(hostValuePart) - } - stmt := strings.TrimSuffix(sb.String(), ",") - _, err := ds.writer.ExecContext(context.Background(), stmt, args...) - require.NoError(b, err) - - swInsert := ` - INSERT INTO software ( - name, - version, - source - ) VALUES ` - swValuePart := `(?, ?, ?),` - - sb.Reset() - sb.WriteString(swInsert) - args = make([]interface{}, 0, sws*3) - for i := 0; i < sws; i++ { - name := "software" + strconv.Itoa(i) - args = append(args, name, strconv.Itoa(i)+".0.0", "testing") - sb.WriteString(swValuePart) - } - stmt = strings.TrimSuffix(sb.String(), ",") - _, err = ds.writer.ExecContext(context.Background(), stmt, args...) - require.NoError(b, err) - - // cartesian product of hosts and software tables - hostSwInsert := ` - INSERT INTO host_software (host_id, software_id) - SELECT - h.id, - sw.id - FROM - hosts h, - software sw` - _, err = ds.writer.ExecContext(context.Background(), hostSwInsert) - require.NoError(b, err) -} - -func resetUpdateAllZeroAgg(b *testing.B, ds *Datastore) { - updateStmt := `UPDATE aggregated_stats SET json_value = CAST(0 AS json) WHERE type = "software_hosts_count"` - _, err := ds.writer.ExecContext(context.Background(), updateStmt) - require.NoError(b, err) -} - -func singleSelectGroupByInsertBatchAgg(b *testing.B, ds *Datastore, updatedAt time.Time, batchSize int) { - queryStmt := ` - SELECT count(*), software_id - FROM host_software - GROUP BY software_id` - - insertStmt := ` - INSERT INTO aggregated_stats - (id, type, json_value, updated_at) - VALUES - %s - ON DUPLICATE KEY UPDATE - json_value = VALUES(json_value), - updated_at = VALUES(updated_at)` - valuesPart := `(?, "software_hosts_count", CAST(? AS json), ?),` - - rows, err := ds.reader.QueryContext(context.Background(), queryStmt) - require.NoError(b, err) - defer rows.Close() - - var batchCount int - args := make([]interface{}, 0, batchSize*3) - for rows.Next() { - var count int - var sid uint - - require.NoError(b, rows.Scan(&count, &sid)) - args = append(args, sid, count, updatedAt) - batchCount++ - - if batchCount == batchSize { - values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") - _, err := ds.writer.ExecContext(context.Background(), fmt.Sprintf(insertStmt, values), args...) - require.NoError(b, err) - - args = args[:0] - batchCount = 0 - } - } - - if batchCount > 0 { - values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",") - _, err := ds.writer.ExecContext(context.Background(), fmt.Sprintf(insertStmt, values), args...) - require.NoError(b, err) - } - require.NoError(b, rows.Err()) -} diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 53809edc5..63146909b 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -508,7 +508,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) { }) t.Run("hosts count", func(t *testing.T) { - defer TruncateTables(t, ds, "aggregated_stats") + defer TruncateTables(t, ds, "software_host_counts") listSoftwareCheckCount(t, ds, 0, 0, fleet.SoftwareListOptions{WithHostCounts: true}, false) // create the counts for those software and re-run diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 222db5027..a4dace831 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -47,11 +47,15 @@ func (s *integrationTestSuite) TearDownTest() { var ids []uint for _, host := range hosts { ids = append(ids, host.ID) + require.NoError(t, s.ds.UpdateHostSoftware(context.Background(), host.ID, nil)) } if len(ids) > 0 { require.NoError(t, s.ds.DeleteHosts(ctx, ids)) } + // recalculate software counts will remove the software entries + require.NoError(t, s.ds.CalculateHostsPerSoftware(context.Background(), time.Now())) + lbls, err := s.ds.ListLabels(ctx, fleet.TeamFilter{}, fleet.ListOptions{}) require.NoError(t, err) for _, lbl := range lbls { @@ -2615,6 +2619,136 @@ func (s *integrationTestSuite) TestQuerySpecs() { assert.Equal(t, uint(3), delBatchResp.Deleted) } +func (s *integrationTestSuite) TestPaginateListSoftware() { + t := s.T() + + // create a few hosts specific to this test + hosts := make([]*fleet.Host, 20) + for i := range hosts { + host, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: t.Name() + strconv.Itoa(i), + OsqueryHostID: t.Name() + strconv.Itoa(i), + UUID: t.Name() + strconv.Itoa(i), + Hostname: t.Name() + "foo" + strconv.Itoa(i) + ".local", + PrimaryIP: "192.168.1." + strconv.Itoa(i), + PrimaryMac: fmt.Sprintf("30-65-EC-6F-C4-%02d", i), + }) + require.NoError(t, err) + require.NotNil(t, host) + hosts[i] = host + } + + // create a bunch of software + sws := make([]fleet.Software, 20) + for i := range sws { + sw := fleet.Software{Name: "sw" + strconv.Itoa(i), Version: "0.0." + strconv.Itoa(i), Source: "apps"} + sws[i] = sw + } + + // mark them as installed on the hosts, with host at index 0 having all 20, + // at index 1 having 19, index 2 = 18, etc. until index 19 = 1. So software + // sws[0] is only used by 1 host, while sws[19] is used by all. + for i, h := range hosts { + require.NoError(t, s.ds.UpdateHostSoftware(context.Background(), h.ID, sws[i:])) + require.NoError(t, s.ds.LoadHostSoftware(context.Background(), h)) + + if i == 0 { + // this host has all software, refresh the list so we have the software.ID filled + sws = h.Software + } + } + + for i, sw := range sws { + cpe := "somecpe" + strconv.Itoa(i) + require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), sw, cpe)) + + if i < 10 { + // add CVEs for the first 10 software, which are the least used (lower hosts_count) + _, err := s.ds.InsertCVEForCPE(context.Background(), fmt.Sprintf("cve-123-123-%03d", i), []string{cpe}) + require.NoError(t, err) + } + } + + assertResp := func(resp listSoftwareResponse, want []fleet.Software, ts time.Time, counts ...int) { + require.Len(t, resp.Software, len(want)) + for i := range resp.Software { + wantID, gotID := want[i].ID, resp.Software[i].ID + assert.Equal(t, wantID, gotID) + wantCount, gotCount := counts[i], resp.Software[i].HostsCount + assert.Equal(t, wantCount, gotCount) + } + if ts.IsZero() { + assert.Nil(t, resp.CountsUpdatedAt) + } else { + require.NotNil(t, resp.CountsUpdatedAt) + assert.WithinDuration(t, ts, *resp.CountsUpdatedAt, time.Second) + } + } + + // no software host counts have been calculated yet, so this returns nothing + var lsResp listSoftwareResponse + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, nil, time.Time{}) + + // calculate hosts counts + hostsCountTs := time.Now().UTC() + require.NoError(t, s.ds.CalculateHostsPerSoftware(context.Background(), hostsCountTs)) + + // now the list software endpoint returns the software, get the first page without vulns + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "0", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, []fleet.Software{sws[19], sws[18], sws[17], sws[16], sws[15]}, hostsCountTs, 20, 19, 18, 17, 16) + + // second page (page=1) + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "1", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, []fleet.Software{sws[14], sws[13], sws[12], sws[11], sws[10]}, hostsCountTs, 15, 14, 13, 12, 11) + + // third page (page=2) + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "2", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, []fleet.Software{sws[9], sws[8], sws[7], sws[6], sws[5]}, hostsCountTs, 10, 9, 8, 7, 6) + + // last page (page=3) + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "3", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, []fleet.Software{sws[4], sws[3], sws[2], sws[1], sws[0]}, hostsCountTs, 5, 4, 3, 2, 1) + + // past the end + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "5", "page", "4", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, nil, time.Time{}) + + // no explicit sort order, defaults to hosts_count DESC + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "2", "page", "0") + assertResp(lsResp, []fleet.Software{sws[19], sws[18]}, hostsCountTs, 20, 19) + + // hosts_count ascending + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "per_page", "3", "page", "0", "order_key", "hosts_count", "order_direction", "asc") + assertResp(lsResp, []fleet.Software{sws[0], sws[1], sws[2]}, hostsCountTs, 1, 2, 3) + + // vulnerable software only + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "per_page", "5", "page", "0", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, []fleet.Software{sws[9], sws[8], sws[7], sws[6], sws[5]}, hostsCountTs, 10, 9, 8, 7, 6) + + // vulnerable software only, next page + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "per_page", "5", "page", "1", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, []fleet.Software{sws[4], sws[3], sws[2], sws[1], sws[0]}, hostsCountTs, 5, 4, 3, 2, 1) + + // vulnerable software only, past last page + lsResp = listSoftwareResponse{} + s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "per_page", "5", "page", "2", "order_key", "hosts_count", "order_direction", "desc") + assertResp(lsResp, nil, time.Time{}) +} + // creates a session and returns it, its key is to be passed as authorization header. func createSession(t *testing.T, uid uint, ds fleet.Datastore) *fleet.Session { key := make([]byte, 64)