Add vulnerable filter for software and also wire up the query search (#2604)

* Add vulnerable filter for software and also wire up the query search

* Add documentation

* Update to use software list options

changes/issue-2049-add-vulnerable-filter-to-software

@@ -0,0 +1 @@
+* Add vulnerable=true/false/1/0 query parameter to the software listing endpoint and wire up the query search for filtering software by name.

cmd/fleetctl/get_test.go

@@ -532,8 +532,8 @@ func TestGetSoftawre(t *testing.T) {
 
 	var gotTeamID *uint
 
-	ds.ListSoftwareFunc = func(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error) {
+	ds.ListSoftwareFunc = func(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
-		gotTeamID = teamId
+		gotTeamID = opt.TeamID
 		return []fleet.Software{foo001, foo002, foo003, bar003}, nil
 	}
 

docs/01-Using-Fleet/03-REST-API.md

@@ -6439,6 +6439,7 @@ If the `name` is not already associated with an existing team, this API route cr
 | order_direction         | string  | query | **Requires `order_key`**. The direction of the order given the order key. Options include `asc` and `desc`. Default is `asc`.                                                                                                                                                                                                               |
 | query                   | string  | query | Search query keywords. Searchable fields include `hostname`, `machine_serial`, `uuid`, and `ipv4`.                                                                                                                                                                                                                                          |
 | team_id                 | integer | query | _Available in Fleet Premium_ Filters the users to only include users in the specified team.                                                                                                                                                                                                                                                 |
+| vulnerable              | bool    | query | If true or 1, only list software that has detected vulnerabilities                                                                                                                                                                                                                                                                          |
 
 #### Example
 

server/datastore/mysql/software.go

@@ -92,7 +92,7 @@ func nothingChanged(current []fleet.Software, incoming []fleet.Software) bool {
 }
 
 func applyChangesForNewSoftwareDB(ctx context.Context, tx sqlx.ExtContext, host *fleet.Host) error {
-	storedCurrentSoftware, err := listSoftwareDB(ctx, tx, &host.ID, nil, fleet.ListOptions{})
+	storedCurrentSoftware, err := listSoftwareDB(ctx, tx, &host.ID, fleet.SoftwareListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "loading current software for host")
 	}
@@ -200,44 +200,49 @@ func insertNewInstalledHostSoftwareDB(
 	return nil
 }
 
-func listSoftwareDB(ctx context.Context, q sqlx.QueryerContext, hostID *uint, teamID *uint, opt fleet.ListOptions) ([]fleet.Software, error) {
+func listSoftwareDB(ctx context.Context, q sqlx.QueryerContext, hostID *uint, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
 	hostWhere := `hs.host_id=?`
 	if hostID == nil {
 		hostWhere = "TRUE"
 	}
 	teamWhere := `h.team_id=?`
-	if teamID == nil {
+	if opt.TeamID == nil {
 		teamWhere = "TRUE"
 	}
+	vulnerableJoin := "LEFT JOIN software_cpe scp ON (s.id=scp.software_id)"
+	if opt.Vulnerable {
+		vulnerableJoin = `JOIN software_cpe scp ON (s.id=scp.software_id)
+		JOIN software_cve scv ON (scp.id=scv.cpe_id)`
+	}
 	sql := fmt.Sprintf(`
 		SELECT DISTINCT s.*, coalesce(scp.cpe, "") as generated_cpe
 		FROM host_software hs
 		JOIN hosts h ON (hs.host_id=h.id)
 		JOIN software s ON (hs.software_id=s.id)
-		LEFT JOIN software_cpe scp ON (s.id=scp.software_id)
+		%s
 		WHERE %s AND %s
-		GROUP BY s.id, s.name, s.version, s.source, generated_cpe
+	`, vulnerableJoin, hostWhere, teamWhere)
-	`, hostWhere, teamWhere)
-	sql = appendListOptionsToSQL(sql, opt)
 
 	var result []*fleet.Software
 	vars := []interface{}{}
 	if hostID != nil {
 		vars = append(vars, hostID)
 	}
-	if teamID != nil {
+	if opt.TeamID != nil {
-		vars = append(vars, teamID)
+		vars = append(vars, opt.TeamID)
 	}
-	if err := sqlx.SelectContext(ctx, q, &result, sql, vars...); err != nil {
+	sql, listVars := searchLike(sql, vars, opt.MatchQuery, "s.name", "s.version")
+	sql += ` GROUP BY s.id, s.name, s.version, s.source, generated_cpe `
+	sql = appendListOptionsToSQL(sql, opt.ListOptions)
+	if err := sqlx.SelectContext(ctx, q, &result, sql, listVars...); err != nil {
 		return nil, errors.Wrap(err, "load host software")
 	}
 
 	sql = fmt.Sprintf(`
-		SELECT DISTINCT s.id, scv.cve
+		SELECT DISTINCT hs.software_id, scv.cve
 		FROM host_software hs
 		JOIN hosts h ON (hs.host_id=h.id)
-		JOIN software s ON (s.id=hs.software_id)
+		JOIN software_cpe scp ON (hs.software_id=scp.software_id)
-		JOIN software_cpe scp ON (s.id=scp.software_id)
 		JOIN software_cve scv ON (scp.id=scv.cpe_id)
 		WHERE %s AND %s
 	`, hostWhere, teamWhere)
@@ -275,7 +280,7 @@ func listSoftwareDB(ctx context.Context, q sqlx.QueryerContext, hostID *uint, te
 
 func (d *Datastore) LoadHostSoftware(ctx context.Context, host *fleet.Host) error {
 	host.HostSoftware = fleet.HostSoftware{Modified: false}
-	software, err := listSoftwareDB(ctx, d.reader, &host.ID, nil, fleet.ListOptions{})
+	software, err := listSoftwareDB(ctx, d.reader, &host.ID, fleet.SoftwareListOptions{})
 	if err != nil {
 		return err
 	}
@@ -358,8 +363,8 @@ func (d *Datastore) InsertCVEForCPE(ctx context.Context, cve string, cpes []stri
 	return nil
 }
 
-func (d *Datastore) ListSoftware(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error) {
+func (d *Datastore) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
-	return listSoftwareDB(ctx, d.reader, nil, teamId, opt)
+	return listSoftwareDB(ctx, d.reader, nil, opt)
 }
 
 func (d *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software, error) {

server/datastore/mysql/software_test.go

@@ -426,7 +426,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 	bar003 := fleet.Software{Name: "bar", Version: "0.0.3", Source: "deb_packages"}
 
 	t.Run("lists everything", func(t *testing.T) {
-		software, err := ds.ListSoftware(context.Background(), nil, fleet.ListOptions{})
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{})
 		require.NoError(t, err)
 
 		require.Len(t, software, 4)
@@ -435,7 +435,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 	})
 
 	t.Run("limits the results", func(t *testing.T) {
-		software, err := ds.ListSoftware(context.Background(), nil, fleet.ListOptions{PerPage: 1, OrderKey: "version"})
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{PerPage: 1, OrderKey: "version"}})
 		require.NoError(t, err)
 
 		require.Len(t, software, 1)
@@ -444,7 +444,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 	})
 
 	t.Run("paginates", func(t *testing.T) {
-		software, err := ds.ListSoftware(context.Background(), nil, fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"})
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}})
 		require.NoError(t, err)
 
 		require.Len(t, software, 1)
@@ -457,7 +457,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		require.NoError(t, err)
 		require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID}))
 
-		software, err := ds.ListSoftware(context.Background(), &team1.ID, fleet.ListOptions{OrderKey: "version"})
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{OrderKey: "version"}, TeamID: &team1.ID})
 		require.NoError(t, err)
 
 		require.Len(t, software, 2)
@@ -470,11 +470,29 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 		require.NoError(t, err)
 		require.NoError(t, ds.AddHostsToTeam(context.Background(), &team1.ID, []uint{host1.ID}))
 
-		software, err := ds.ListSoftware(context.Background(), &team1.ID, fleet.ListOptions{PerPage: 1, Page: 1, OrderKey: "id"})
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{PerPage: 1, Page: 1, OrderKey: "id"}, TeamID: &team1.ID})
 		require.NoError(t, err)
 
 		require.Len(t, software, 1)
 		expected := []fleet.Software{foo003}
 		test.ElementsMatchSkipID(t, software, expected)
 	})
+
+	t.Run("filters vulnerable software", func(t *testing.T) {
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{Vulnerable: true})
+		require.NoError(t, err)
+
+		require.Len(t, software, 1)
+		expected := []fleet.Software{foo001}
+		test.ElementsMatchSkipID(t, software, expected)
+	})
+
+	t.Run("filters by query", func(t *testing.T) {
+		software, err := ds.ListSoftware(context.Background(), fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{MatchQuery: "bar"}})
+		require.NoError(t, err)
+
+		require.Len(t, software, 1)
+		expected := []fleet.Software{bar003}
+		test.ElementsMatchSkipID(t, software, expected)
+	})
 }

server/fleet/app.go

@@ -241,18 +241,18 @@ const (
 // listing objects
 type ListOptions struct {
 	// Which page to return (must be positive integer)
-	Page uint
+	Page uint `query:"page,optional"`
 	// How many results per page (must be positive integer, 0 indicates
 	// unlimited)
-	PerPage uint
+	PerPage uint `query:"per_page,optional"`
 	// Key to use for ordering
-	OrderKey string
+	OrderKey string `query:"order_key,optional"`
 	// Direction of ordering
-	OrderDirection OrderDirection
+	OrderDirection OrderDirection `query:"order_direction,optional"`
 	// MatchQuery is the query string to match against columns of the entity
 	// (varies depending on entity, eg. hostname, IP address for hosts).
 	// Handling for this parameter must be implemented separately for each type.
-	MatchQuery string
+	MatchQuery string `query:"query,optional"`
 }
 
 type ListQueryOptions struct {

server/fleet/datastore.go

@@ -360,7 +360,7 @@ type Datastore interface {
 	// MigrationStatus returns nil if migrations are complete, and an error if migrations need to be run.
 	MigrationStatus(ctx context.Context) (MigrationStatus, error)
 
-	ListSoftware(ctx context.Context, teamId *uint, opt ListOptions) ([]Software, error)
+	ListSoftware(ctx context.Context, opt SoftwareListOptions) ([]Software, error)
 
 	///////////////////////////////////////////////////////////////////////////////
 	// Team Policies

server/fleet/service.go

@@ -408,7 +408,7 @@ type Service interface {
 	///////////////////////////////////////////////////////////////////////////////
 	// Software
 
-	ListSoftware(ctx context.Context, teamID *uint, opt ListOptions) ([]Software, error)
+	ListSoftware(ctx context.Context, opt SoftwareListOptions) ([]Software, error)
 	SoftwareByID(ctx context.Context, id uint) (*Software, error)
 
 	///////////////////////////////////////////////////////////////////////////////

server/fleet/software.go

@@ -46,3 +46,10 @@ type SoftwareIterator interface {
 	Err() error
 	Close() error
 }
+
+type SoftwareListOptions struct {
+	ListOptions
+
+	TeamID     *uint `query:"team_id,optional"`
+	Vulnerable bool  `query:"vulnerable,optional"`
+}

server/mock/datastore_mock.go

@@ -291,7 +291,7 @@ type MigrateDataFunc func(ctx context.Context) error
 
 type MigrationStatusFunc func(ctx context.Context) (fleet.MigrationStatus, error)
 
-type ListSoftwareFunc func(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error)
+type ListSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error)
 
 type NewTeamPolicyFunc func(ctx context.Context, teamID uint, queryID uint, resolution string) (*fleet.Policy, error)
 
@@ -1448,9 +1448,9 @@ func (s *DataStore) MigrationStatus(ctx context.Context) (fleet.MigrationStatus,
 	return s.MigrationStatusFunc(ctx)
 }
 
-func (s *DataStore) ListSoftware(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error) {
+func (s *DataStore) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
 	s.ListSoftwareFuncInvoked = true
-	return s.ListSoftwareFunc(ctx, teamId, opt)
+	return s.ListSoftwareFunc(ctx, opt)
 }
 
 func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, queryID uint, resolution string) (*fleet.Policy, error) {

server/service/endpoint_utils.go

@@ -148,14 +148,14 @@ func makeDecoder(iface interface{}) kithttp.DecodeRequestFunc {
 					return nil, err
 				}
 				queryVal := r.URL.Query().Get(queryTagValue)
-				if field.Kind() == reflect.Ptr {
+				// if optional and it's a ptr, leave as nil
-					// if optional and it's a ptr, leave as nil
+				if queryVal == "" {
-					if queryVal == "" {
+					if optional {
-						if optional {
+						continue
-							continue
-						}
-						return nil, errors.Errorf("Param %s is required", f.Name)
 					}
+					return nil, errors.Errorf("Param %s is required", f.Name)
+				}
+				if field.Kind() == reflect.Ptr {
 					// create the new instance of whatever it is
 					field.Set(reflect.New(field.Type().Elem()))
 					field = field.Elem()
@@ -169,6 +169,8 @@ func makeDecoder(iface interface{}) kithttp.DecodeRequestFunc {
 						return nil, errors.Wrap(err, "parsing uint from query")
 					}
 					field.SetUint(uint64(queryValUint))
+				case reflect.Bool:
+					field.SetBool(queryVal == "1" || queryVal == "true")
 				default:
 					return nil, errors.Errorf("Cant handle type for field %s %s", f.Name, field.Kind())
 				}

server/service/integration_core_test.go

@@ -281,7 +281,7 @@ func (s *integrationTestSuite) TestVulnerableSoftware() {
 		SeenTime:        time.Now(),
 		NodeKey:         t.Name() + "1",
 		UUID:            t.Name() + "1",
-		Hostname:        "foo.local",
+		Hostname:        t.Name() + "foo.local",
 		PrimaryIP:       "192.168.1.1",
 		PrimaryMac:      "30-65-EC-6F-C4-58",
 	})
@@ -330,6 +330,13 @@ func (s *integrationTestSuite) TestVulnerableSoftware() {
 	// ignoring other things like timestamps and things that are outside the cope of this ticket
 	assert.Contains(t, string(bodyBytes), expectedJSONSoft2)
 	assert.Contains(t, string(bodyBytes), expectedJSONSoft1)
+
+	lsReq := listSoftwareRequest{}
+	lsResp := listSoftwareResponse{}
+	s.DoJSON("GET", "/api/v1/fleet/software", lsReq, http.StatusOK, &lsResp, "vulnerable", "true")
+	assert.Len(t, lsResp.Software, 1)
+	assert.Equal(t, soft1.ID, lsResp.Software[0].ID)
+	assert.Len(t, lsResp.Software[0].Vulnerabilities, 1)
 }
 
 func (s *integrationTestSuite) TestGlobalPolicies() {

server/service/software.go

@@ -11,8 +11,7 @@ import (
 /////////////////////////////////////////////////////////////////////////////////
 
 type listSoftwareRequest struct {
-	TeamID      *uint             `query:"team_id,optional"`
+	fleet.SoftwareListOptions
-	ListOptions fleet.ListOptions `url:"list_options"`
 }
 
 type listSoftwareResponse struct {
@@ -24,17 +23,17 @@ func (r listSoftwareResponse) error() error { return r.Err }
 
 func listSoftwareEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (interface{}, error) {
 	req := request.(*listSoftwareRequest)
-	resp, err := svc.ListSoftware(ctx, req.TeamID, req.ListOptions)
+	resp, err := svc.ListSoftware(ctx, req.SoftwareListOptions)
 	if err != nil {
 		return listSoftwareResponse{Err: err}, nil
 	}
 	return listSoftwareResponse{Software: resp}, nil
 }
 
-func (svc Service) ListSoftware(ctx context.Context, teamID *uint, opt fleet.ListOptions) ([]fleet.Software, error) {
+func (svc Service) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
 	if err := svc.authz.Authorize(ctx, &fleet.Software{}, fleet.ActionRead); err != nil {
 		return nil, err
 	}
 
-	return svc.ds.ListSoftware(ctx, teamID, opt)
+	return svc.ds.ListSoftware(ctx, opt)
 }

server/service/software_test.go

@@ -16,9 +16,9 @@ func TestService_ListSoftware(t *testing.T) {
 	ds := new(mock.Store)
 
 	var calledWithTeamID *uint
-	var calledWithOpt fleet.ListOptions
+	var calledWithOpt fleet.SoftwareListOptions
-	ds.ListSoftwareFunc = func(ctx context.Context, teamId *uint, opt fleet.ListOptions) ([]fleet.Software, error) {
+	ds.ListSoftwareFunc = func(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
-		calledWithTeamID = teamId
+		calledWithTeamID = opt.TeamID
 		calledWithOpt = opt
 		return []fleet.Software{}, nil
 	}
@@ -29,10 +29,10 @@ func TestService_ListSoftware(t *testing.T) {
 	ctx := context.Background()
 	ctx = viewer.NewContext(ctx, viewer.Viewer{User: user})
 
-	_, err := svc.ListSoftware(ctx, ptr.Uint(42), fleet.ListOptions{PerPage: 77, Page: 4})
+	_, err := svc.ListSoftware(ctx, fleet.SoftwareListOptions{TeamID: ptr.Uint(42), ListOptions: fleet.ListOptions{PerPage: 77, Page: 4}})
 	require.NoError(t, err)
 
 	assert.True(t, ds.ListSoftwareFuncInvoked)
 	assert.Equal(t, ptr.Uint(42), calledWithTeamID)
-	assert.Equal(t, fleet.ListOptions{PerPage: 77, Page: 4}, calledWithOpt)
+	assert.Equal(t, fleet.ListOptions{PerPage: 77, Page: 4}, calledWithOpt.ListOptions)
 }