Add host count to software API (#2879)

* Add host count to software API

* Update docs

* Update fleetctl tests to account for host counts

* Update docs to mention host_count special case

* Update func comment

changes/add-host-count-to-software-api

@@ -0,0 +1 @@
+* Add host_count to the software API

cmd/fleetctl/get_test.go

@@ -527,10 +527,11 @@ func TestGetSoftawre(t *testing.T) {
 			{CVE: "cve-321-432-543", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-321-432-543"},
 			{CVE: "cve-333-444-555", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-333-444-555"},
 		},
+		HostCount: 2,
 	}
-	foo002 := fleet.Software{Name: "foo", Version: "0.0.2", Source: "chrome_extensions"}
+	foo002 := fleet.Software{Name: "foo", Version: "0.0.2", Source: "chrome_extensions", HostCount: 1}
-	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns"}
+	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns", HostCount: 43}
-	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages", BundleIdentifier: "bundle"}
+	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages", BundleIdentifier: "bundle", HostCount: 9}
 
 	var gotTeamID *uint
 
@@ -557,6 +558,7 @@ apiVersion: "1"
 kind: software
 spec:
 - generated_cpe: somecpe
+  host_count: 2
   id: 0
   name: foo
   source: chrome_extensions
@@ -567,12 +569,14 @@ spec:
   - cve: cve-333-444-555
     details_link: https://nvd.nist.gov/vuln/detail/cve-333-444-555
 - generated_cpe: ""
+  host_count: 1
   id: 0
   name: foo
   source: chrome_extensions
   version: 0.0.2
   vulnerabilities: null
 - generated_cpe: someothercpewithoutvulns
+  host_count: 43
   id: 0
   name: foo
   source: chrome_extensions
@@ -580,13 +584,14 @@ spec:
   vulnerabilities: null
 - bundle_identifier: bundle
   generated_cpe: ""
+  host_count: 9
   id: 0
   name: bar
   source: deb_packages
   version: 0.0.3
   vulnerabilities: null
 `
-	expectedJson := `{"kind":"software","apiVersion":"1","spec":[{"id":0,"name":"foo","version":"0.0.1","source":"chrome_extensions","generated_cpe":"somecpe","vulnerabilities":[{"cve":"cve-321-432-543","details_link":"https://nvd.nist.gov/vuln/detail/cve-321-432-543"},{"cve":"cve-333-444-555","details_link":"https://nvd.nist.gov/vuln/detail/cve-333-444-555"}]},{"id":0,"name":"foo","version":"0.0.2","source":"chrome_extensions","generated_cpe":"","vulnerabilities":null},{"id":0,"name":"foo","version":"0.0.3","source":"chrome_extensions","generated_cpe":"someothercpewithoutvulns","vulnerabilities":null},{"id":0,"name":"bar","version":"0.0.3","bundle_identifier":"bundle","source":"deb_packages","generated_cpe":"","vulnerabilities":null}]}
+	expectedJson := `{"kind":"software","apiVersion":"1","spec":[{"id":0,"name":"foo","version":"0.0.1","source":"chrome_extensions","generated_cpe":"somecpe","vulnerabilities":[{"cve":"cve-321-432-543","details_link":"https://nvd.nist.gov/vuln/detail/cve-321-432-543"},{"cve":"cve-333-444-555","details_link":"https://nvd.nist.gov/vuln/detail/cve-333-444-555"}],"host_count":2},{"id":0,"name":"foo","version":"0.0.2","source":"chrome_extensions","generated_cpe":"","vulnerabilities":null,"host_count":1},{"id":0,"name":"foo","version":"0.0.3","source":"chrome_extensions","generated_cpe":"someothercpewithoutvulns","vulnerabilities":null,"host_count":43},{"id":0,"name":"bar","version":"0.0.3","bundle_identifier":"bundle","source":"deb_packages","generated_cpe":"","vulnerabilities":null,"host_count":9}]}
 `
 
 	assert.Equal(t, expected, runAppForTest(t, []string{"get", "software"}))

docs/01-Using-Fleet/03-REST-API.md

@@ -645,6 +645,8 @@ Returns the information of the specified host.
 
 The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../02-Deploying/02-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature.
 
+The host_count parameter in the software list will always be 1 in this call, as the view of the software list is within this host. On other APIs, such as `/api/v1/fleet/software` with a broader scope, it counts within that scope.
+
 `GET /api/v1/fleet/hosts/{id}`
 
 #### Parameters
@@ -673,7 +675,8 @@ The endpoint returns the host's installed `software` if the software inventory f
         "version": "4.5.1",
         "source": "rpm_packages",
         "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 1
       },
       {
         "id": 1146,
@@ -681,7 +684,8 @@ The endpoint returns the host's installed `software` if the software inventory f
         "version": "1.30",
         "source": "rpm_packages",
         "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 1
       },
       {
         "id": 321,
@@ -690,7 +694,8 @@ The endpoint returns the host's installed `software` if the software inventory f
         "source": "apps",
         "bundle_identifier": "com.some.app",
         "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 1
       }
     ],
     "id": 1,
@@ -5303,7 +5308,8 @@ _Available in Fleet Premium_
         "version": "2.1.11",
         "source": "Application (macOS)",
         "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 2
       },
       {
         "id": 2,
@@ -5311,7 +5317,8 @@ _Available in Fleet Premium_
         "version": "2.1.11",
         "source": "Application (macOS)",
         "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 22
       },
       {
         "id": 3,
@@ -5319,7 +5326,8 @@ _Available in Fleet Premium_
         "version": "2.1.11",
         "source": "rpm_packages",
         "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 5
       },
       {
         "id": 4,
@@ -5327,8 +5335,9 @@ _Available in Fleet Premium_
         "version": "2.1.11",
         "source": "rpm_packages",
         "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
-      },
+        "host_count": 9
+      }
     ]
   }
 }

server/datastore/mysql/software.go

@@ -215,9 +215,10 @@ var dialect = goqu.Dialect("mysql")
 func listSoftwareDB(
 	ctx context.Context, q sqlx.QueryerContext, hostID *uint, opts fleet.SoftwareListOptions,
 ) ([]fleet.Software, error) {
-	ds := dialect.From(goqu.I("host_software").As("hs")).SelectDistinct(
+	ds := dialect.From(goqu.I("host_software").As("hs")).Select(
 		"s.*",
 		goqu.COALESCE(goqu.I("scp.cpe"), "").As("generated_cpe"),
+		goqu.COUNT(goqu.DISTINCT("hs.host_id")).As("host_count"),
 	).Join(
 		goqu.I("hosts").As("h"),
 		goqu.On(

server/datastore/mysql/software_test.go

@@ -69,16 +69,18 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {
 
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
 	assert.False(t, host1.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft1.Software, host1.HostSoftware.Software)
 
 	soft1ByID, err := ds.SoftwareByID(context.Background(), host1.HostSoftware.Software[0].ID)
 	require.NoError(t, err)
 	require.NotNil(t, soft1ByID)
+	// SoftwareByID does not calculate HostCount
+	soft1ByID.HostCount = 1
 	assert.Equal(t, host1.HostSoftware.Software[0], *soft1ByID)
 
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)
 
 	soft1 = fleet.HostSoftware{
 		Modified: true,
@@ -100,11 +102,11 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {
 
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
 	assert.False(t, host1.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft1.Software, host1.HostSoftware.Software)
 
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)
 
 	soft1 = fleet.HostSoftware{
 		Modified: true,
@@ -119,7 +121,7 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {
 
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
 	assert.False(t, host1.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft1.Software, host1.HostSoftware.Software)
 
 	soft2 = fleet.HostSoftware{
 		Modified: true,
@@ -134,7 +136,7 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {
 	require.NoError(t, ds.SaveHostSoftware(context.Background(), host2))
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)
 
 	soft2 = fleet.HostSoftware{
 		Modified: true,
@@ -149,7 +151,7 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {
 	require.NoError(t, ds.SaveHostSoftware(context.Background(), host2))
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)
 }
 
 func testSoftwareCPE(t *testing.T, ds *Datastore) {
@@ -452,10 +454,11 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 			{CVE: "cve-321-432-543", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-321-432-543"},
 			{CVE: "cve-333-444-555", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-333-444-555"},
 		},
+		HostCount: 1,
 	}
-	foo002 := fleet.Software{Name: "foo", Version: "v0.0.2", Source: "chrome_extensions"}
+	foo002 := fleet.Software{Name: "foo", Version: "v0.0.2", Source: "chrome_extensions", HostCount: 1}
-	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns"}
+	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns", HostCount: 2}
-	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages"}
+	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages", HostCount: 1}
 
 	t.Run("lists everything", func(t *testing.T) {
 		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{})
@@ -493,7 +496,12 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		require.NoError(t, err)
 
 		require.Len(t, software, 2)
-		expected := []fleet.Software{foo001, foo003}
+		// Counts differ because we are only counting the hosts in the team
+		foo001WithCount := foo001
+		foo001WithCount.HostCount = 1
+		foo003WithCount := foo003
+		foo003WithCount.HostCount = 1
+		expected := []fleet.Software{foo001WithCount, foo003WithCount}
 		test.ElementsMatchSkipID(t, software, expected)
 	})
 
@@ -513,7 +521,10 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		require.NoError(t, err)
 
 		require.Len(t, software, 1)
-		expected := []fleet.Software{foo003}
+
+		foo003WithCount := foo003
+		foo003WithCount.HostCount = 1
+		expected := []fleet.Software{foo003WithCount}
 		test.ElementsMatchSkipID(t, software, expected)
 	})
 
@@ -546,4 +557,14 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		expected = []fleet.Software{foo002}
 		test.ElementsMatchSkipID(t, software, expected)
 	})
+
+	t.Run("can order by host count", func(t *testing.T) {
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "host_count", OrderDirection: fleet.OrderDescending}})
+		require.NoError(t, err)
+
+		require.Len(t, software, 4)
+		software[0].Name = foo003.Name
+		software[0].Version = foo003.Version
+		software[0].Source = foo003.Source
+	})
 }

server/fleet/software.go

@@ -21,6 +21,9 @@ type Software struct {
 	GenerateCPE string `json:"generated_cpe" db:"generated_cpe"`
 	// Vulnerabilities lists all the found CVEs for the CPE
 	Vulnerabilities VulnerabilitiesSlice `json:"vulnerabilities"`
+
+	// HostCount is the amount of hosts that currently have this software installed
+	HostCount int `json:"host_count" db:"host_count"`
 }
 
 func (Software) AuthzType() string {

server/service/integration_core_test.go

@@ -337,6 +337,13 @@ func (s *integrationTestSuite) TestVulnerableSoftware() {
 	assert.Len(t, lsResp.Software, 1)
 	assert.Equal(t, soft1.ID, lsResp.Software[0].ID)
 	assert.Len(t, lsResp.Software[0].Vulnerabilities, 1)
+	assert.Equal(t, 1, lsResp.Software[0].HostCount)
+
+	s.DoJSON("GET", "/api/v1/fleet/software", lsReq, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "host_count", "order_direction", "desc")
+	assert.Len(t, lsResp.Software, 1)
+	assert.Equal(t, soft1.ID, lsResp.Software[0].ID)
+	assert.Len(t, lsResp.Software[0].Vulnerabilities, 1)
+	assert.Equal(t, 1, lsResp.Software[0].HostCount)
 }
 
 func (s *integrationTestSuite) TestGlobalPolicies() {

server/test/comparisons.go

@@ -34,6 +34,25 @@ func ElementsMatchSkipID(t TestingT, listA, listB interface{}, msgAndArgs ...int
 	return ElementsMatchWithOptions(t, listA, listB, []cmp.Option{opt}, msgAndArgs)
 }
 
+// ElementsMatchSkipIDAndHostCount asserts that the elements match, skipping any field with
+// name "ID" or "HostCount".
+func ElementsMatchSkipIDAndHostCount(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) {
+	t.Helper()
+
+	opt := cmp.FilterPath(func(p cmp.Path) bool {
+		for _, ps := range p {
+			switch ps := ps.(type) {
+			case cmp.StructField:
+				if ps.Name() == "ID" || ps.Name() == "HostCount" {
+					return true
+				}
+			}
+		}
+		return false
+	}, cmp.Ignore())
+	return ElementsMatchWithOptions(t, listA, listB, []cmp.Option{opt}, msgAndArgs)
+}
+
 // ElementsMatchSkipTimestampsID asserts that the elements match, skipping any field with
 // name "ID", "CreatedAt", and "UpdatedAt". This is useful for comparing after DB insertion.
 func ElementsMatchSkipTimestampsID(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) {