Add host count to software API (#2879)

* Add host count to software API * Update docs * Update fleetctl tests to account for host counts * Update docs to mention host_count special case * Update func comment
2024-11-06 08:55:24 +00:00 · 2021-11-11 08:49:17 -03:00 · 2021-11-11 08:49:17 -03:00 · b802af6f44
commit b802af6f44
parent 88b32d8c7f
8 changed files with 91 additions and 25 deletions
--- a/changes/add-host-count-to-software-api
+++ b/changes/add-host-count-to-software-api
@ -0,0 +1 @@
+* Add host_count to the software API
--- a/cmd/fleetctl/get_test.go
+++ b/cmd/fleetctl/get_test.go
@ -527,10 +527,11 @@ func TestGetSoftawre(t *testing.T) {
 			{CVE: "cve-321-432-543", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-321-432-543"},
 			{CVE: "cve-333-444-555", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-333-444-555"},
 		},
+		HostCount: 2,
 	}
-	foo002 := fleet.Software{Name: "foo", Version: "0.0.2", Source: "chrome_extensions"}
-	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns"}
-	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages", BundleIdentifier: "bundle"}
+	foo002 := fleet.Software{Name: "foo", Version: "0.0.2", Source: "chrome_extensions", HostCount: 1}
+	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns", HostCount: 43}
+	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages", BundleIdentifier: "bundle", HostCount: 9}

 	var gotTeamID *uint

@ -557,6 +558,7 @@ apiVersion: "1"
 kind: software
 spec:
 - generated_cpe: somecpe
+  host_count: 2
  id: 0
  name: foo
  source: chrome_extensions
@ -567,12 +569,14 @@ spec:
  - cve: cve-333-444-555
    details_link: https://nvd.nist.gov/vuln/detail/cve-333-444-555
 - generated_cpe: ""
+  host_count: 1
  id: 0
  name: foo
  source: chrome_extensions
  version: 0.0.2
  vulnerabilities: null
 - generated_cpe: someothercpewithoutvulns
+  host_count: 43
  id: 0
  name: foo
  source: chrome_extensions
@ -580,13 +584,14 @@ spec:
  vulnerabilities: null
 - bundle_identifier: bundle
  generated_cpe: ""
+  host_count: 9
  id: 0
  name: bar
  source: deb_packages
  version: 0.0.3
  vulnerabilities: null
 `
-	expectedJson := `{"kind":"software","apiVersion":"1","spec":[{"id":0,"name":"foo","version":"0.0.1","source":"chrome_extensions","generated_cpe":"somecpe","vulnerabilities":[{"cve":"cve-321-432-543","details_link":"https://nvd.nist.gov/vuln/detail/cve-321-432-543"},{"cve":"cve-333-444-555","details_link":"https://nvd.nist.gov/vuln/detail/cve-333-444-555"}]},{"id":0,"name":"foo","version":"0.0.2","source":"chrome_extensions","generated_cpe":"","vulnerabilities":null},{"id":0,"name":"foo","version":"0.0.3","source":"chrome_extensions","generated_cpe":"someothercpewithoutvulns","vulnerabilities":null},{"id":0,"name":"bar","version":"0.0.3","bundle_identifier":"bundle","source":"deb_packages","generated_cpe":"","vulnerabilities":null}]}
+	expectedJson := `{"kind":"software","apiVersion":"1","spec":[{"id":0,"name":"foo","version":"0.0.1","source":"chrome_extensions","generated_cpe":"somecpe","vulnerabilities":[{"cve":"cve-321-432-543","details_link":"https://nvd.nist.gov/vuln/detail/cve-321-432-543"},{"cve":"cve-333-444-555","details_link":"https://nvd.nist.gov/vuln/detail/cve-333-444-555"}],"host_count":2},{"id":0,"name":"foo","version":"0.0.2","source":"chrome_extensions","generated_cpe":"","vulnerabilities":null,"host_count":1},{"id":0,"name":"foo","version":"0.0.3","source":"chrome_extensions","generated_cpe":"someothercpewithoutvulns","vulnerabilities":null,"host_count":43},{"id":0,"name":"bar","version":"0.0.3","bundle_identifier":"bundle","source":"deb_packages","generated_cpe":"","vulnerabilities":null,"host_count":9}]}
 `

 	assert.Equal(t, expected, runAppForTest(t, []string{"get", "software"}))
--- a/docs/01-Using-Fleet/03-REST-API.md
+++ b/docs/01-Using-Fleet/03-REST-API.md
@ -645,6 +645,8 @@ Returns the information of the specified host.

 The endpoint returns the host's installed `software` if the software inventory feature flag is turned on. This feature flag is turned off by default. [Check out the feature flag documentation](../02-Deploying/02-Configuration.md#feature-flags) for instructions on how to turn on the software inventory feature.

+The host_count parameter in the software list will always be 1 in this call, as the view of the software list is within this host. On other APIs, such as `/api/v1/fleet/software` with a broader scope, it counts within that scope.
+
 `GET /api/v1/fleet/hosts/{id}`

 #### Parameters
@ -673,7 +675,8 @@ The endpoint returns the host's installed `software` if the software inventory f
        "version": "4.5.1",
        "source": "rpm_packages",
        "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 1
      },
      {
        "id": 1146,
@ -681,7 +684,8 @@ The endpoint returns the host's installed `software` if the software inventory f
        "version": "1.30",
        "source": "rpm_packages",
        "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 1
      },
      {
        "id": 321,
@ -690,7 +694,8 @@ The endpoint returns the host's installed `software` if the software inventory f
        "source": "apps",
        "bundle_identifier": "com.some.app",
        "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 1
      }
    ],
    "id": 1,
@ -5303,7 +5308,8 @@ _Available in Fleet Premium_
        "version": "2.1.11",
        "source": "Application (macOS)",
        "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 2
      },
      {
        "id": 2,
@ -5311,7 +5317,8 @@ _Available in Fleet Premium_
        "version": "2.1.11",
        "source": "Application (macOS)",
        "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 22
      },
      {
        "id": 3,
@ -5319,7 +5326,8 @@ _Available in Fleet Premium_
        "version": "2.1.11",
        "source": "rpm_packages",
        "generated_cpe": "",
-        "vulnerabilities": null
+        "vulnerabilities": null,
+        "host_count": 5
      },
      {
        "id": 4,
@ -5327,8 +5335,9 @@ _Available in Fleet Premium_
        "version": "2.1.11",
        "source": "rpm_packages",
        "generated_cpe": "",
-        "vulnerabilities": null
-      },
+        "vulnerabilities": null,
+        "host_count": 9
+      }
    ]
  }
 }
--- a/server/datastore/mysql/software.go
+++ b/server/datastore/mysql/software.go
@ -215,9 +215,10 @@ var dialect = goqu.Dialect("mysql")
 func listSoftwareDB(
 	ctx context.Context, q sqlx.QueryerContext, hostID *uint, opts fleet.SoftwareListOptions,
 ) ([]fleet.Software, error) {
-	ds := dialect.From(goqu.I("host_software").As("hs")).SelectDistinct(
+	ds := dialect.From(goqu.I("host_software").As("hs")).Select(
 		"s.*",
 		goqu.COALESCE(goqu.I("scp.cpe"), "").As("generated_cpe"),
+		goqu.COUNT(goqu.DISTINCT("hs.host_id")).As("host_count"),
 	).Join(
 		goqu.I("hosts").As("h"),
 		goqu.On(
--- a/server/datastore/mysql/software_test.go
+++ b/server/datastore/mysql/software_test.go
@ -69,16 +69,18 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {

 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
 	assert.False(t, host1.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft1.Software, host1.HostSoftware.Software)

 	soft1ByID, err := ds.SoftwareByID(context.Background(), host1.HostSoftware.Software[0].ID)
 	require.NoError(t, err)
 	require.NotNil(t, soft1ByID)
+	// SoftwareByID does not calculate HostCount
+	soft1ByID.HostCount = 1
 	assert.Equal(t, host1.HostSoftware.Software[0], *soft1ByID)

 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)

 	soft1 = fleet.HostSoftware{
 		Modified: true,
@ -100,11 +102,11 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {

 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
 	assert.False(t, host1.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft1.Software, host1.HostSoftware.Software)

 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)

 	soft1 = fleet.HostSoftware{
 		Modified: true,
@ -119,7 +121,7 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {

 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host1))
 	assert.False(t, host1.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft1.Software, host1.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft1.Software, host1.HostSoftware.Software)

 	soft2 = fleet.HostSoftware{
 		Modified: true,
@ -134,7 +136,7 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {
 	require.NoError(t, ds.SaveHostSoftware(context.Background(), host2))
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)

 	soft2 = fleet.HostSoftware{
 		Modified: true,
@ -149,7 +151,7 @@ func testSoftwareSaveHost(t *testing.T, ds *Datastore) {
 	require.NoError(t, ds.SaveHostSoftware(context.Background(), host2))
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host2))
 	assert.False(t, host2.HostSoftware.Modified)
-	test.ElementsMatchSkipID(t, soft2.Software, host2.HostSoftware.Software)
+	test.ElementsMatchSkipIDAndHostCount(t, soft2.Software, host2.HostSoftware.Software)
 }

 func testSoftwareCPE(t *testing.T, ds *Datastore) {
@ -452,10 +454,11 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 			{CVE: "cve-321-432-543", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-321-432-543"},
 			{CVE: "cve-333-444-555", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-333-444-555"},
 		},
+		HostCount: 1,
 	}
-	foo002 := fleet.Software{Name: "foo", Version: "v0.0.2", Source: "chrome_extensions"}
-	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns"}
-	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages"}
+	foo002 := fleet.Software{Name: "foo", Version: "v0.0.2", Source: "chrome_extensions", HostCount: 1}
+	foo003 := fleet.Software{Name: "foo", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "someothercpewithoutvulns", HostCount: 2}
+	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages", HostCount: 1}

 	t.Run("lists everything", func(t *testing.T) {
 		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{})
@ -493,7 +496,12 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		require.NoError(t, err)

 		require.Len(t, software, 2)
-		expected := []fleet.Software{foo001, foo003}
+		// Counts differ because we are only counting the hosts in the team
+		foo001WithCount := foo001
+		foo001WithCount.HostCount = 1
+		foo003WithCount := foo003
+		foo003WithCount.HostCount = 1
+		expected := []fleet.Software{foo001WithCount, foo003WithCount}
 		test.ElementsMatchSkipID(t, software, expected)
 	})

@ -513,7 +521,10 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		require.NoError(t, err)

 		require.Len(t, software, 1)
-		expected := []fleet.Software{foo003}
+
+		foo003WithCount := foo003
+		foo003WithCount.HostCount = 1
+		expected := []fleet.Software{foo003WithCount}
 		test.ElementsMatchSkipID(t, software, expected)
 	})

@ -546,4 +557,14 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		expected = []fleet.Software{foo002}
 		test.ElementsMatchSkipID(t, software, expected)
 	})
+
+	t.Run("can order by host count", func(t *testing.T) {
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "host_count", OrderDirection: fleet.OrderDescending}})
+		require.NoError(t, err)
+
+		require.Len(t, software, 4)
+		software[0].Name = foo003.Name
+		software[0].Version = foo003.Version
+		software[0].Source = foo003.Source
+	})
 }
--- a/server/fleet/software.go
+++ b/server/fleet/software.go
@ -21,6 +21,9 @@ type Software struct {
 	GenerateCPE string `json:"generated_cpe" db:"generated_cpe"`
 	// Vulnerabilities lists all the found CVEs for the CPE
 	Vulnerabilities VulnerabilitiesSlice `json:"vulnerabilities"`
+
+	// HostCount is the amount of hosts that currently have this software installed
+	HostCount int `json:"host_count" db:"host_count"`
 }

 func (Software) AuthzType() string {
--- a/server/service/integration_core_test.go
+++ b/server/service/integration_core_test.go
@ -337,6 +337,13 @@ func (s *integrationTestSuite) TestVulnerableSoftware() {
 	assert.Len(t, lsResp.Software, 1)
 	assert.Equal(t, soft1.ID, lsResp.Software[0].ID)
 	assert.Len(t, lsResp.Software[0].Vulnerabilities, 1)
+	assert.Equal(t, 1, lsResp.Software[0].HostCount)
+
+	s.DoJSON("GET", "/api/v1/fleet/software", lsReq, http.StatusOK, &lsResp, "vulnerable", "true", "order_key", "host_count", "order_direction", "desc")
+	assert.Len(t, lsResp.Software, 1)
+	assert.Equal(t, soft1.ID, lsResp.Software[0].ID)
+	assert.Len(t, lsResp.Software[0].Vulnerabilities, 1)
+	assert.Equal(t, 1, lsResp.Software[0].HostCount)
 }

 func (s *integrationTestSuite) TestGlobalPolicies() {
--- a/server/test/comparisons.go
+++ b/server/test/comparisons.go
@ -34,6 +34,25 @@ func ElementsMatchSkipID(t TestingT, listA, listB interface{}, msgAndArgs ...int
 	return ElementsMatchWithOptions(t, listA, listB, []cmp.Option{opt}, msgAndArgs)
 }

+// ElementsMatchSkipIDAndHostCount asserts that the elements match, skipping any field with
+// name "ID" or "HostCount".
+func ElementsMatchSkipIDAndHostCount(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) {
+	t.Helper()
+
+	opt := cmp.FilterPath(func(p cmp.Path) bool {
+		for _, ps := range p {
+			switch ps := ps.(type) {
+			case cmp.StructField:
+				if ps.Name() == "ID" || ps.Name() == "HostCount" {
+					return true
+				}
+			}
+		}
+		return false
+	}, cmp.Ignore())
+	return ElementsMatchWithOptions(t, listA, listB, []cmp.Option{opt}, msgAndArgs)
+}
+
 // ElementsMatchSkipTimestampsID asserts that the elements match, skipping any field with
 // name "ID", "CreatedAt", and "UpdatedAt". This is useful for comparing after DB insertion.
 func ElementsMatchSkipTimestampsID(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) {