empayre/fleet
14
0
Fork 0
mirror of https://github.com/empayre/fleet.git synced 2024-11-06 08:55:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add hosts_count field to "list software" endpoint (#3873)

Browse Source
This commit is contained in:
Martin Angers 2022-01-26 09:47:56 -05:00 committed by GitHub
parent 39b34508a9
commit 9a0f749641
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 476 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changes/issue-3086-add-hosts-count-to-software Normal file
View File

@ -0,0 +1 @@
* Add `hosts_count` field for each software (and `counts_updated_at` timestamp as a top-level field) to the response payload of `GET /api/v1/fleet/software`

46
cmd/fleet/serve.go
View File

@ -643,10 +643,12 @@ func cronVulnerabilities(
level.Error(logger).Log("config", "couldn't read app config", "err", err)
return
}
vulnDisabled := false
if appConfig.VulnerabilitySettings.DatabasesPath == "" &&
config.Vulnerabilities.DatabasesPath == "" {
level.Info(logger).Log("vulnerability scanning", "not configured")
return
vulnDisabled = true
}
if !appConfig.HostSettings.EnableSoftwareInventory {
level.Info(logger).Log("software inventory", "not configured")
@ -664,15 +666,19 @@ func cronVulnerabilities(
"result", vulnPath)
}
level.Info(logger).Log("databases-path", vulnPath)
if !vulnDisabled {
level.Info(logger).Log("databases-path", vulnPath)
}
level.Info(logger).Log("periodicity", config.Vulnerabilities.Periodicity)
if config.Vulnerabilities.CurrentInstanceChecks == "auto" {
level.Debug(logger).Log("current instance checks", "auto", "trying to create databases-path", vulnPath)
err := os.MkdirAll(vulnPath, 0o755)
if err != nil {
level.Error(logger).Log("databases-path", "creation failed, returning", "err", err)
return
if !vulnDisabled {
if config.Vulnerabilities.CurrentInstanceChecks == "auto" {
level.Debug(logger).Log("current instance checks", "auto", "trying to create databases-path", vulnPath)
err := os.MkdirAll(vulnPath, 0o755)
if err != nil {
level.Error(logger).Log("databases-path", "creation failed, returning", "err", err)
return
}
}
}
@ -694,16 +700,24 @@ func cronVulnerabilities(
}
}
err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config)
if err != nil {
level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err)
sentry.CaptureException(err)
continue
if !vulnDisabled {
err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config)
if err != nil {
level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err)
sentry.CaptureException(err)
continue
}
err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config)
if err != nil {
level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err)
sentry.CaptureException(err)
continue
}
}
err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config)
if err != nil {
level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err)
if err := ds.CalculateHostsPerSoftware(ctx, time.Now()); err != nil {
level.Error(logger).Log("msg", "calculating hosts count per software", "err", err)
sentry.CaptureException(err)
continue
}

53
docs/01-Using-Fleet/03-REST-API.md
View File

@ -1063,7 +1063,7 @@ Request (`filters` is specified):
Requires the [macadmins osquery
extension](https://github.com/macadmins/osquery-extension) which comes bundled in [Fleet's osquery
installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
Retrieves a host's Google Chrome profile information which can be used to link a host to a specific
user by email.
@ -1098,11 +1098,11 @@ user by email.
---
### Get host's mobile device management (MDM) and Munki information
### Get host's mobile device management (MDM) and Munki information
Requires the [macadmins osquery
extension](https://github.com/macadmins/osquery-extension) which comes bundled in [Fleet's osquery
installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
installers](https://fleetdm.com/docs/using-fleet/adding-hosts#osquery-installer).
Retrieves a host's MDM enrollment status, MDM server URL, and Munki version.
@ -5872,10 +5872,10 @@ Transforms a host name into a host id. For example, the Fleet UI use this endpoi
| ----------------------- | ------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| page | integer | query | Page number of the results to fetch. |
| per_page | integer | query | Results per page. |
| order_key | string | query | What to order results by. Can be ordered by the following fields: `name`. |
| order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default is `asc`. |
| query | string | query | Search query keywords. Searchable fields include `name`. |
| team_id | integer | query | _Available in Fleet Premium_ Filters the software to only include the software installed on the hosts that are assigned to the specified team. |
| order_key | string | query | What to order results by. Can be ordered by the following fields: `name`, `hosts_count`. Defaults to the hosts count, descending. |
| order_direction | string | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default if not provided is `asc`. |
| query | string | query | Search query keywords. Searchable fields include `name`. |
| team_id | integer | query | _Available in Fleet Premium_ Filters the software to only include the software installed on the hosts that are assigned to the specified team. |
| vulnerable | bool | query | If true or 1, only list software that has detected vulnerabilities |
#### Example
@ -5888,22 +5888,16 @@ Transforms a host name into a host id. For example, the Fleet UI use this endpoi
```json
{
"counts_updated_at": "2022-01-01 12:32:00",
"software": [
{
"id": 1,
"name": "Chrome.app",
"id": 4,
"name": "osquery",
"version": "2.1.11",
"source": "Application (macOS)",
"source": "rpm_packages",
"generated_cpe": "",
"vulnerabilities": null
},
{
"id": 2,
"name": "Figma.app",
"version": "2.1.11",
"source": "Application (macOS)",
"generated_cpe": "",
"vulnerabilities": null
"vulnerabilities": null,
"hosts_count": 456
},
{
"id": 3,
@ -5911,15 +5905,26 @@ Transforms a host name into a host id. For example, the Fleet UI use this endpoi
"version": "2.1.11",
"source": "rpm_packages",
"generated_cpe": "",
"vulnerabilities": null
"vulnerabilities": null,
"hosts_count": 345
},
{
"id": 4,
"name": "osquery",
"id": 2,
"name": "Figma.app",
"version": "2.1.11",
"source": "rpm_packages",
"source": "Application (macOS)",
"generated_cpe": "",
"vulnerabilities": null
"vulnerabilities": null,
"hosts_count": 234
},
{
"id": 1,
"name": "Chrome.app",
"version": "2.1.11",
"source": "Application (macOS)",
"generated_cpe": "",
"vulnerabilities": null,
"hosts_count": 123
}
]
}

2
docs/02-Deploying/03-Configuration.md
View File

@ -1846,7 +1846,7 @@ When `current_instance_checks` is set to `auto` (the default), Fleet instances w
##### periodicity
How often vulnerabilities are checked.
How often vulnerabilities are checked. This is also the interval at which the counts of hosts per software is calculated.
- Default value: `1h`
- Environment variable: `FLEET_VULNERABILITIES_PERIODICITY`

1
go.sum
View File

@ -1051,6 +1051,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY=

25
server/datastore/mysql/migrations/tables/20220125105650_UpdateAggregatedStats.go Normal file
View File

@ -0,0 +1,25 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20220125105650, Down_20220125105650)
}
func Up_20220125105650(tx *sql.Tx) error {
if _, err := tx.Exec(`ALTER TABLE aggregated_stats MODIFY id bigint(20) unsigned NOT NULL`); err != nil {
return errors.Wrap(err, "make aggregated_stats.id bigint")
}
if _, err := tx.Exec("create index aggregated_stats_type_idx on aggregated_stats(`type`);"); err != nil {
return errors.Wrap(err, "creating aggregated_stats index")
}
return nil
}
func Down_20220125105650(tx *sql.Tx) error {
return nil
}

9
server/datastore/mysql/schema.sql
View File

File diff suppressed because one or more lines are too long

98
server/datastore/mysql/software.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"sort"
"strings"
"time"
"github.com/doug-martin/goqu/v9"
_ "github.com/doug-martin/goqu/v9/dialect/mysql"
@ -306,6 +307,19 @@ func selectSoftwareSQL(hostID *uint, opts fleet.SoftwareListOptions) (string, []
)
}
if opts.WithHostCounts {
ds = ds.Join(
goqu.I("aggregated_stats").As("shc"),
goqu.On(
goqu.I("s.id").Eq(goqu.I("shc.id")),
),
).Where(goqu.I("shc.type").Eq("software_hosts_count"), goqu.I("shc.json_value").Gt(0)).
SelectAppend(
goqu.I("shc.json_value").As("hosts_count"),
goqu.I("shc.updated_at").As("counts_updated_at"),
)
}
return ds.ToSQL()
}
@ -519,3 +533,87 @@ func (d *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software,
return &software, nil
}
// CalculateHostsPerSoftware calculates the number of hosts having each
// software installed and stores that information in the aggregated_stats
// table.
func (d *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error {
// NOTE(mna): for reference, on my laptop I get ~1.5ms for 10_000 hosts / 100 software each,
// ~1.5s for 10_000 hosts / 1_000 software each (but this is with an otherwise empty
// aggregated_stats table, but still reasonable numbers give that this runs as a cron
// task in the background).
resetStmt := `
UPDATE aggregated_stats
SET json_value = CAST(0 AS json)
WHERE type = "software_hosts_count"`
queryStmt := `
SELECT count(*), software_id
FROM host_software
GROUP BY software_id`
insertStmt := `
INSERT INTO aggregated_stats
(id, type, json_value, updated_at)
VALUES
%s
ON DUPLICATE KEY UPDATE
json_value = VALUES(json_value),
updated_at = VALUES(updated_at)`
valuesPart := `(?, "software_hosts_count", CAST(? AS json), ?),`
// first, reset all counts to 0
if _, err := d.writer.ExecContext(ctx, resetStmt); err != nil {
return ctxerr.Wrap(ctx, err, "reset all software_hosts_count to 0 in aggregated_stats")
}
// next get a cursor for the counts for each software
rows, err := d.reader.QueryContext(ctx, queryStmt)
if err != nil {
return ctxerr.Wrap(ctx, err, "read counts from host_software")
}
defer rows.Close()
// use a loop to iterate to prevent loading all in one go in memory, as it
// could get pretty big at >100K hosts with 1000+ software each.
const batchSize = 100
var batchCount int
args := make([]interface{}, 0, batchSize*3)
for rows.Next() {
var count int
var sid uint
if err := rows.Scan(&count, &sid); err != nil {
return ctxerr.Wrap(ctx, err, "scan row into variables")
}
args = append(args, sid, count, updatedAt)
batchCount++
if batchCount == batchSize {
values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",")
if _, err := d.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil {
return ctxerr.Wrap(ctx, err, "insert batch into aggregated_stats")
}
args = args[:0]
batchCount = 0
}
}
if batchCount > 0 {
values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",")
if _, err := d.writer.ExecContext(ctx, fmt.Sprintf(insertStmt, values), args...); err != nil {
return ctxerr.Wrap(ctx, err, "insert batch into aggregated_stats")
}
}
if err := rows.Err(); err != nil {
return ctxerr.Wrap(ctx, err, "iterate over host_software counts")
}
// TODO(mna): delete any unused software from the software table (any that
// isn't in that list with a host count > 0). This also addresses another
// TODO in this file.
return nil
}

170
server/datastore/mysql/software_bench_test.go Normal file
View File

@ -0,0 +1,170 @@
package mysql
import (
"context"
"fmt"
"strconv"
"strings"
"testing"
"time"
"github.com/fleetdm/fleet/v4/server"
"github.com/stretchr/testify/require"
)
func BenchmarkCalculateHostsPerSoftware(b *testing.B) {
ts := time.Now()
type counts struct{ hs, sws int }
cases := []counts{
{1, 1},
{10, 10},
{100, 100},
{1_000, 100},
{10_000, 100},
{10_000, 1_000},
}
b.Run("resetUpdate", func(b *testing.B) {
b.Run("singleSelectGroupByInsertBatch100AggStats", func(b *testing.B) {
for _, c := range cases {
b.Run(fmt.Sprintf("%d:%d", c.hs, c.sws), func(b *testing.B) {
ds := CreateMySQLDS(b)
generateHostsWithSoftware(b, ds, c.hs, c.sws)
b.ResetTimer()
for i := 0; i < b.N; i++ {
resetUpdateAllZeroAgg(b, ds)
singleSelectGroupByInsertBatchAgg(b, ds, ts, 100)
}
checkCountsAgg(b, ds, c.hs, c.sws)
})
}
})
})
}
func checkCountsAgg(b *testing.B, ds *Datastore, hs, sws int) {
var rowsCount, invalidHostsCount int
rowsStmt := `SELECT COUNT(*) FROM aggregated_stats WHERE type = "software_hosts_count"`
err := ds.writer.GetContext(context.Background(), &rowsCount, rowsStmt)
require.NoError(b, err)
require.Equal(b, sws, rowsCount)
invalidStmt := `SELECT COUNT(*) FROM aggregated_stats WHERE type = "software_hosts_count" AND json_value != CAST(? AS json)`
err = ds.writer.GetContext(context.Background(), &invalidHostsCount, invalidStmt, hs)
require.NoError(b, err)
require.Equal(b, 0, invalidHostsCount)
}
func generateHostsWithSoftware(b *testing.B, ds *Datastore, hs, sws int) {
hostInsert := `
INSERT INTO hosts (
osquery_host_id,
node_key,
hostname,
uuid
)
VALUES `
hostValuePart := `(?, ?, ?, ?),`
var sb strings.Builder
sb.WriteString(hostInsert)
args := make([]interface{}, 0, hs*4)
for i := 0; i < hs; i++ {
osqueryHostID, _ := server.GenerateRandomText(10)
name := "host" + strconv.Itoa(i)
args = append(args, osqueryHostID, name+"key", name, name+"uuid")
sb.WriteString(hostValuePart)
}
stmt := strings.TrimSuffix(sb.String(), ",")
_, err := ds.writer.ExecContext(context.Background(), stmt, args...)
require.NoError(b, err)
swInsert := `
INSERT INTO software (
name,
version,
source
) VALUES `
swValuePart := `(?, ?, ?),`
sb.Reset()
sb.WriteString(swInsert)
args = make([]interface{}, 0, sws*3)
for i := 0; i < sws; i++ {
name := "software" + strconv.Itoa(i)
args = append(args, name, strconv.Itoa(i)+".0.0", "testing")
sb.WriteString(swValuePart)
}
stmt = strings.TrimSuffix(sb.String(), ",")
_, err = ds.writer.ExecContext(context.Background(), stmt, args...)
require.NoError(b, err)
// cartesian product of hosts and software tables
hostSwInsert := `
INSERT INTO host_software (host_id, software_id)
SELECT
h.id,
sw.id
FROM
hosts h,
software sw`
_, err = ds.writer.ExecContext(context.Background(), hostSwInsert)
require.NoError(b, err)
}
func resetUpdateAllZeroAgg(b *testing.B, ds *Datastore) {
updateStmt := `UPDATE aggregated_stats SET json_value = CAST(0 AS json) WHERE type = "software_hosts_count"`
_, err := ds.writer.ExecContext(context.Background(), updateStmt)
require.NoError(b, err)
}
func singleSelectGroupByInsertBatchAgg(b *testing.B, ds *Datastore, updatedAt time.Time, batchSize int) {
queryStmt := `
SELECT count(*), software_id
FROM host_software
GROUP BY software_id`
insertStmt := `
INSERT INTO aggregated_stats
(id, type, json_value, updated_at)
VALUES
%s
ON DUPLICATE KEY UPDATE
json_value = VALUES(json_value),
updated_at = VALUES(updated_at)`
valuesPart := `(?, "software_hosts_count", CAST(? AS json), ?),`
rows, err := ds.reader.QueryContext(context.Background(), queryStmt)
require.NoError(b, err)
defer rows.Close()
var batchCount int
args := make([]interface{}, 0, batchSize*3)
for rows.Next() {
var count int
var sid uint
require.NoError(b, rows.Scan(&count, &sid))
args = append(args, sid, count, updatedAt)
batchCount++
if batchCount == batchSize {
values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",")
_, err := ds.writer.ExecContext(context.Background(), fmt.Sprintf(insertStmt, values), args...)
require.NoError(b, err)
args = args[:0]
batchCount = 0
}
}
if batchCount > 0 {
values := strings.TrimSuffix(strings.Repeat(valuesPart, batchCount), ",")
_, err := ds.writer.ExecContext(context.Background(), fmt.Sprintf(insertStmt, values), args...)
require.NoError(b, err)
}
require.NoError(b, rows.Err())
}

43
server/datastore/mysql/software_test.go
View File

@ -401,19 +401,19 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages"}
t.Run("lists everything", func(t *testing.T) {
software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{})
software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{}, true)
expected := []fleet.Software{bar003, foo001, foo003, foo002}
test.ElementsMatchSkipID(t, software, expected)
})
t.Run("limits the results", func(t *testing.T) {
software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{PerPage: 1, OrderKey: "version"}})
software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{PerPage: 1, OrderKey: "version"}}, true)
expected := []fleet.Software{foo001}
test.ElementsMatchSkipID(t, software, expected)
})
t.Run("paginates", func(t *testing.T) {
software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}})
software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}}, true)
expected := []fleet.Software{foo003}
test.ElementsMatchSkipID(t, software, expected)
})
@ -423,7 +423,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
require.NoError(t, err)
require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID}))
software := listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "version"}, TeamID: &team1.ID})
software := listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "version"}, TeamID: &team1.ID}, true)
expected := []fleet.Software{foo001, foo003}
test.ElementsMatchSkipID(t, software, expected)
})
@ -440,34 +440,34 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
OrderKey: "id",
},
TeamID: &team1.ID,
})
}, true)
expected := []fleet.Software{foo003}
test.ElementsMatchSkipID(t, software, expected)
})
t.Run("filters vulnerable software", func(t *testing.T) {
software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{VulnerableOnly: true})
software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{VulnerableOnly: true}, true)
expected := []fleet.Software{foo001}
test.ElementsMatchSkipID(t, software, expected)
})
t.Run("filters by query", func(t *testing.T) {
// query by name (case insensitive)
software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "baR"}})
software := listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "baR"}}, true)
expected := []fleet.Software{bar003}
test.ElementsMatchSkipID(t, software, expected)
// query by version
software = listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "0.0.3"}})
software = listSoftwareCheckCount(t, ds, 2, 2, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "0.0.3"}}, true)
expected = []fleet.Software{foo003, bar003}
test.ElementsMatchSkipID(t, software, expected)
// query by version (case insensitive)
software = listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "V0.0.2"}})
software = listSoftwareCheckCount(t, ds, 1, 1, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "V0.0.2"}}, true)
expected = []fleet.Software{foo002}
test.ElementsMatchSkipID(t, software, expected)
})
t.Run("can order by name and id", func(t *testing.T) {
software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "name,id", OrderDirection: fleet.OrderAscending}})
software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "name,id", OrderDirection: fleet.OrderAscending}}, false)
assert.Equal(t, bar003.Name, software[0].Name)
assert.Equal(t, bar003.Version, software[0].Version)
assert.Equal(t, bar003.Source, software[0].Source)
@ -476,9 +476,21 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
assert.Greater(t, software[2].ID, software[1].ID)
assert.Greater(t, software[3].ID, software[2].ID)
})
t.Run("hosts count", func(t *testing.T) {
defer TruncateTables(t, ds, "aggregated_stats")
listSoftwareCheckCount(t, ds, 0, 0, fleet.SoftwareListOptions{WithHostCounts: true}, false)
// create the counts for those software and re-run
require.NoError(t, ds.CalculateHostsPerSoftware(context.Background(), time.Now()))
software := listSoftwareCheckCount(t, ds, 4, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, WithHostCounts: true}, false)
// ordered by counts descending, so foo003 is first
assert.Equal(t, foo003.Name, software[0].Name)
assert.Equal(t, 2, software[0].HostsCount)
})
}
func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareListOptions) []fleet.Software {
func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int, expectedFullCount int, opts fleet.SoftwareListOptions, returnSorted bool) []fleet.Software {
software, err := ds.ListSoftware(context.Background(), opts)
require.NoError(t, err)
require.Len(t, software, expectedListCount)
@ -490,8 +502,11 @@ func listSoftwareCheckCount(t *testing.T, ds *Datastore, expectedListCount int,
return s.Vulnerabilities[i].CVE < s.Vulnerabilities[j].CVE
})
}
sort.Slice(software, func(i, j int) bool {
return software[i].Name+software[i].Version < software[j].Name+software[j].Version
})
if returnSorted {
sort.Slice(software, func(i, j int) bool {
return software[i].Name+software[i].Version < software[j].Name+software[j].Version
})
}
return software
}

5
server/datastore/mysql/testing_utils.go
View File

@ -243,6 +243,11 @@ func createMySQLDSWithOptions(t testing.TB, opts *DatastoreTestOptions) *Datasto
strings.TrimPrefix(details.Name(), "github.com/fleetdm/fleet/v4/"), "/", "_",
)
cleanName = strings.ReplaceAll(cleanName, ".", "_")
if len(cleanName) > 60 {
// the later parts are more unique than the start, with the package names,
// so trim from the start.
cleanName = cleanName[len(cleanName)-60:]
}
ds := initializeDatabase(t, cleanName, opts)
t.Cleanup(func() { ds.Close() })
return ds

1
server/fleet/datastore.go
View File

@ -326,6 +326,7 @@ type Datastore interface {
AllCPEs(ctx context.Context) ([]string, error)
InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error
SoftwareByID(ctx context.Context, id uint) (*Software, error)
CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error
///////////////////////////////////////////////////////////////////////////////
// ActivitiesStore

13
server/fleet/software.go
View File

@ -1,5 +1,7 @@
package fleet
import "time"
type SoftwareCVE struct {
CVE string `json:"cve" db:"cve"`
DetailsLink string `json:"details_link" db:"details_link"`
@ -21,6 +23,12 @@ type Software struct {
GenerateCPE string `json:"generated_cpe" db:"generated_cpe"`
// Vulnerabilities lists all the found CVEs for the CPE
Vulnerabilities VulnerabilitiesSlice `json:"vulnerabilities"`
// HostsCount indicates the number of hosts with that software, filled only
// if explicitly requested.
HostsCount int `json:"hosts_count,omitempty" db:"hosts_count"`
// CountsUpdatedAt is the timestamp when the hosts count was last updated
// for that software, filled only if hosts count is requested.
CountsUpdatedAt time.Time `json:"-" db:"counts_updated_at"`
}
func (Software) AuthzType() string {
@ -54,4 +62,9 @@ type SoftwareListOptions struct {
VulnerableOnly bool `query:"vulnerable,optional"`
SkipLoadingCVEs bool
// WithHostCounts indicates that the list of software should include the
// counts of hosts per software, and include only those software that have
// a count of hosts > 0.
WithHostCounts bool
}

10
server/mock/datastore_mock.go
View File

@ -266,6 +266,8 @@ type InsertCVEForCPEFunc func(ctx context.Context, cve string, cpes []string) er
type SoftwareByIDFunc func(ctx context.Context, id uint) (*fleet.Software, error)
type CalculateHostsPerSoftwareFunc func(ctx context.Context, updatedAt time.Time) error
type NewActivityFunc func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error
type ListActivitiesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Activity, error)
@ -742,6 +744,9 @@ type DataStore struct {
SoftwareByIDFunc SoftwareByIDFunc
SoftwareByIDFuncInvoked bool
CalculateHostsPerSoftwareFunc CalculateHostsPerSoftwareFunc
CalculateHostsPerSoftwareFuncInvoked bool
NewActivityFunc NewActivityFunc
NewActivityFuncInvoked bool
@ -1519,6 +1524,11 @@ func (s *DataStore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software,
return s.SoftwareByIDFunc(ctx, id)
}
func (s *DataStore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error {
s.CalculateHostsPerSoftwareFuncInvoked = true
return s.CalculateHostsPerSoftwareFunc(ctx, updatedAt)
}
func (s *DataStore) NewActivity(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error {
s.NewActivityFuncInvoked = true
return s.NewActivityFunc(ctx, user, activityType, details)

31
server/service/integration_core_test.go
View File

@ -403,20 +403,35 @@ func (s *integrationTestSuite) TestVulnerableSoftware() {
s.DoJSON("GET", "/api/v1/fleet/software/count", countReq, http.StatusOK, &countResp)
assert.Equal(t, 2, countResp.Count)
lsReq := listSoftwareRequest{}
lsResp := listSoftwareResponse{}
s.DoJSON("GET", "/api/v1/fleet/software", lsReq, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc")
assert.Len(t, lsResp.Software, 1)
assert.Equal(t, soft1.ID, lsResp.Software[0].ID)
assert.Len(t, lsResp.Software[0].Vulnerabilities, 1)
// no software host counts have been calculated yet, so this returns nothing
var lsResp listSoftwareResponse
s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc")
require.Len(t, lsResp.Software, 0)
assert.True(t, lsResp.CountsUpdatedAt.IsZero())
// the software/count endpoint is different, it doesn't care about hosts counts
s.DoJSON("GET", "/api/v1/fleet/software/count", countReq, http.StatusOK, &countResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc")
assert.Equal(t, 1, countResp.Count)
s.DoJSON("GET", "/api/v1/fleet/software", lsReq, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "name,id", "order_direction", "desc")
assert.Len(t, lsResp.Software, 1)
// calculate hosts counts
hostsCountTs := time.Now().UTC()
require.NoError(t, s.ds.CalculateHostsPerSoftware(context.Background(), hostsCountTs))
// now the list software endpoint returns the software
s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc")
require.Len(t, lsResp.Software, 1)
assert.Equal(t, soft1.ID, lsResp.Software[0].ID)
assert.Len(t, lsResp.Software[0].Vulnerabilities, 1)
assert.WithinDuration(t, hostsCountTs, lsResp.CountsUpdatedAt, time.Second)
// the count endpoint still returns 1
s.DoJSON("GET", "/api/v1/fleet/software/count", countReq, http.StatusOK, &countResp, "vulnerable", "true", "order_key", "generated_cpe", "order_direction", "desc")
assert.Equal(t, 1, countResp.Count)
// default sort, not only vulnerable
s.DoJSON("GET", "/api/v1/fleet/software", nil, http.StatusOK, &lsResp)
require.True(t, len(lsResp.Software) >= len(software))
assert.WithinDuration(t, hostsCountTs, lsResp.CountsUpdatedAt, time.Second)
}
func (s *integrationTestSuite) TestGlobalPolicies() {

21
server/service/software.go
View File

@ -2,6 +2,7 @@ package service
import (
"context"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
)
@ -15,8 +16,9 @@ type listSoftwareRequest struct {
}
type listSoftwareResponse struct {
Software []fleet.Software `json:"software,omitempty"`
Err error `json:"error,omitempty"`
CountsUpdatedAt time.Time `json:"counts_updated_at,omitempty"`
Software []fleet.Software `json:"software,omitempty"`
Err error `json:"error,omitempty"`
}
func (r listSoftwareResponse) error() error { return r.Err }
@ -27,7 +29,14 @@ func listSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Se
if err != nil {
return listSoftwareResponse{Err: err}, nil
}
return listSoftwareResponse{Software: resp}, nil
var latest time.Time
for _, sw := range resp {
if !sw.CountsUpdatedAt.IsZero() && sw.CountsUpdatedAt.After(latest) {
latest = sw.CountsUpdatedAt
}
}
return listSoftwareResponse{CountsUpdatedAt: latest, Software: resp}, nil
}
func (svc Service) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
@ -35,6 +44,12 @@ func (svc Service) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptio
return nil, err
}
// default sort order to hosts_count descending
if opt.OrderKey == "" {
opt.OrderKey = "hosts_count"
opt.OrderDirection = fleet.OrderDescending
}
opt.WithHostCounts = true
return svc.ds.ListSoftware(ctx, opt)
}

14
server/service/software_test.go
View File

@ -34,5 +34,17 @@ func TestService_ListSoftware(t *testing.T) {
assert.True(t, ds.ListSoftwareFuncInvoked)
assert.Equal(t, ptr.Uint(42), calledWithTeamID)
assert.Equal(t, fleet.ListOptions{PerPage: 77, Page: 4}, calledWithOpt.ListOptions)
// sort order defaults to hosts_count descending, automatically, if not explicitly provided
assert.Equal(t, fleet.ListOptions{PerPage: 77, Page: 4, OrderKey: "hosts_count", OrderDirection: fleet.OrderDescending}, calledWithOpt.ListOptions)
assert.True(t, calledWithOpt.WithHostCounts)
// call again, this time with an explicit sort
ds.ListSoftwareFuncInvoked = false
_, err = svc.ListSoftware(ctx, fleet.SoftwareListOptions{TeamID: nil, ListOptions: fleet.ListOptions{PerPage: 11, Page: 2, OrderKey: "id", OrderDirection: fleet.OrderAscending}})
require.NoError(t, err)
assert.True(t, ds.ListSoftwareFuncInvoked)
assert.Nil(t, calledWithTeamID)
assert.Equal(t, fleet.ListOptions{PerPage: 11, Page: 2, OrderKey: "id", OrderDirection: fleet.OrderAscending}, calledWithOpt.ListOptions)
assert.True(t, calledWithOpt.WithHostCounts)
}

8
server/test/new_objects.go
View File

@ -98,7 +98,7 @@ func AddAllHostsLabel(t *testing.T, ds fleet.Datastore) {
require.NoError(t, err)
}
func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now time.Time) *fleet.Host {
func NewHost(tb testing.TB, ds fleet.Datastore, name, ip, key, uuid string, now time.Time) *fleet.Host {
osqueryHostID, _ := server.GenerateRandomText(10)
h, err := ds.NewHost(context.Background(), &fleet.Host{
Hostname: name,
@ -112,9 +112,9 @@ func NewHost(t *testing.T, ds fleet.Datastore, name, ip, key, uuid string, now t
Platform: "darwin",
})
require.NoError(t, err)
require.NotZero(t, h.ID)
require.NoError(t, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, now))
require.NoError(tb, err)
require.NotZero(tb, h.ID)
require.NoError(tb, ds.MarkHostsSeen(context.Background(), []uint{h.ID}, now))
return h
}