Add refetch host API (#767)

This allows the host details to be refetched on the next check in,
rather than waiting for the normal interval to go by. Associated UI
changes are in-progress.

- Migration and service methods for requesting refetch.
- Expose refetch over API.
- Change detail query logic to respect this flag.

server/datastore/mysql/hosts.go

@@ -89,7 +89,8 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
 			additional = COALESCE(?, additional),
 			enroll_secret_name = ?,
 			primary_ip = ?,
-			primary_mac = ?
+			primary_mac = ?,
+			refetch_requested = ?
 		WHERE id = ?
 	`
 	_, err := d.db.Exec(sqlStatement,
@@ -124,6 +125,7 @@ func (d *Datastore) SaveHost(host *kolide.Host) error {
 		host.EnrollSecretName,
 		host.PrimaryIP,
 		host.PrimaryMac,
+		host.RefetchRequested,
 		host.ID,
 	)
 	if err != nil {
@@ -314,7 +316,8 @@ func (d *Datastore) ListHosts(opt kolide.HostListOptions) ([]*kolide.Host, error
         primary_mac, 
         label_update_time, 
         enroll_secret_name,
-		`
+		refetch_requested,
+	`
 
 	var params []interface{}
 
@@ -531,7 +534,8 @@ func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
 			config_tls_refresh,
 			primary_ip,
 			primary_mac,
-			enroll_secret_name
+			enroll_secret_name,
+			refetch_requested
 		FROM hosts
 		WHERE node_key = ?
 		LIMIT 1

server/datastore/mysql/migrations/tables/20210513115729_RefetchHostDetail.go

@@ -0,0 +1,26 @@
+package tables
+
+import (
+	"database/sql"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20210513115729, Down_20210513115729)
+}
+
+func Up_20210513115729(tx *sql.Tx) error {
+	sql := `
+		ALTER TABLE hosts
+		ADD COLUMN refetch_requested TINYINT(1) NOT NULL DEFAULT 0
+	`
+	if _, err := tx.Exec(sql); err != nil {
+		return errors.Wrap(err, "add column refetch_requested")
+	}
+	return nil
+}
+
+func Down_20210513115729(tx *sql.Tx) error {
+	return nil
+}

server/kolide/hosts.go

@@ -84,6 +84,8 @@ type HostService interface {
 	// Possible matches can be on osquery_host_identifier, node_key, UUID, or
 	// hostname.
 	HostByIdentifier(ctx context.Context, identifier string) (*HostDetail, error)
+	// RefetchHost requests a refetch of host details for the provided host.
+	RefetchHost(ctx context.Context, id uint) (err error)
 
 	FlushSeenHosts(ctx context.Context) error
 }
@@ -112,6 +114,7 @@ type Host struct {
 	LabelUpdateTime  time.Time     `json:"label_updated_at" db:"label_update_time"`   // Time that the host details were last updated
 	LastEnrollTime   time.Time     `json:"last_enrolled_at" db:"last_enroll_time"`    // Time that the host last enrolled
 	SeenTime         time.Time     `json:"seen_time" db:"seen_time"`                  // Time that the host was last "seen"
+	RefetchRequested bool          `json:"refetch_requested" db:"refetch_requested"`
 	NodeKey          string        `json:"-" db:"node_key"`
 	HostName         string        `json:"hostname" db:"host_name"` // there is a fulltext index on this field
 	UUID             string        `json:"uuid" db:"uuid"`          // there is a fulltext index on this field

server/service/endpoint_hosts.go

@@ -188,3 +188,28 @@ func makeDeleteHostEndpoint(svc kolide.Service) endpoint.Endpoint {
 		return deleteHostResponse{}, nil
 	}
 }
+
+////////////////////////////////////////////////////////////////////////////////
+// Refetch Host
+////////////////////////////////////////////////////////////////////////////////
+
+type refetchHostRequest struct {
+	ID uint `json:"id"`
+}
+
+type refetchHostResponse struct {
+	Err error `json:"error,omitempty"`
+}
+
+func (r refetchHostResponse) error() error { return r.Err }
+
+func makeRefetchHostEndpoint(svc kolide.Service) endpoint.Endpoint {
+	return func(ctx context.Context, request interface{}) (interface{}, error) {
+		req := request.(refetchHostRequest)
+		err := svc.RefetchHost(ctx, req.ID)
+		if err != nil {
+			return refetchHostResponse{Err: err}, nil
+		}
+		return refetchHostResponse{}, nil
+	}
+}

server/service/handler.go

@@ -94,6 +94,7 @@ type KolideEndpoints struct {
 	GetHost                               endpoint.Endpoint
 	HostByIdentifier                      endpoint.Endpoint
 	DeleteHost                            endpoint.Endpoint
+	RefetchHost                           endpoint.Endpoint
 	ListHosts                             endpoint.Endpoint
 	GetHostSummary                        endpoint.Endpoint
 	SearchTargets                         endpoint.Endpoint
@@ -193,6 +194,7 @@ func MakeKolideServerEndpoints(svc kolide.Service, jwtKey, urlPrefix string, lim
 		ListHosts:                             authenticatedUser(jwtKey, svc, makeListHostsEndpoint(svc)),
 		GetHostSummary:                        authenticatedUser(jwtKey, svc, makeGetHostSummaryEndpoint(svc)),
 		DeleteHost:                            authenticatedUser(jwtKey, svc, makeDeleteHostEndpoint(svc)),
+		RefetchHost:                           authenticatedUser(jwtKey, svc, makeRefetchHostEndpoint(svc)),
 		CreateLabel:                           authenticatedUser(jwtKey, svc, makeCreateLabelEndpoint(svc)),
 		ModifyLabel:                           authenticatedUser(jwtKey, svc, makeModifyLabelEndpoint(svc)),
 		GetLabel:                              authenticatedUser(jwtKey, svc, makeGetLabelEndpoint(svc)),
@@ -305,6 +307,7 @@ type kolideHandlers struct {
 	GetHost                               http.Handler
 	HostByIdentifier                      http.Handler
 	DeleteHost                            http.Handler
+	RefetchHost                           http.Handler
 	ListHosts                             http.Handler
 	GetHostSummary                        http.Handler
 	SearchTargets                         http.Handler
@@ -401,6 +404,7 @@ func makeKolideKitHandlers(e KolideEndpoints, opts []kithttp.ServerOption) *koli
 		GetHost:                               newServer(e.GetHost, decodeGetHostRequest),
 		HostByIdentifier:                      newServer(e.HostByIdentifier, decodeHostByIdentifierRequest),
 		DeleteHost:                            newServer(e.DeleteHost, decodeDeleteHostRequest),
+		RefetchHost:                           newServer(e.RefetchHost, decodeRefetchHostRequest),
 		ListHosts:                             newServer(e.ListHosts, decodeListHostsRequest),
 		GetHostSummary:                        newServer(e.GetHostSummary, decodeNoParamsRequest),
 		SearchTargets:                         newServer(e.SearchTargets, decodeSearchTargetsRequest),
@@ -614,6 +618,7 @@ func attachKolideAPIRoutes(r *mux.Router, h *kolideHandlers) {
 	r.Handle("/api/v1/fleet/hosts/{id}", h.GetHost).Methods("GET").Name("get_host")
 	r.Handle("/api/v1/fleet/hosts/identifier/{identifier}", h.HostByIdentifier).Methods("GET").Name("host_by_identifier")
 	r.Handle("/api/v1/fleet/hosts/{id}", h.DeleteHost).Methods("DELETE").Name("delete_host")
+	r.Handle("/api/v1/fleet/hosts/{id}/refetch", h.RefetchHost).Methods("POST").Name("refetch_host")
 
 	r.Handle("/api/v1/fleet/spec/osquery_options", h.ApplyOsqueryOptionsSpec).Methods("POST").Name("apply_osquery_options_spec")
 	r.Handle("/api/v1/fleet/spec/osquery_options", h.GetOsqueryOptionsSpec).Methods("GET").Name("get_osquery_options_spec")

server/service/service_hosts.go

@@ -75,3 +75,17 @@ func (svc *service) FlushSeenHosts(ctx context.Context) error {
 	hostIDs := svc.seenHostSet.getAndClearHostIDs()
 	return svc.ds.MarkHostsSeen(hostIDs, svc.clock.Now())
 }
+
+func (svc *service) RefetchHost(ctx context.Context, id uint) error {
+	host, err := svc.ds.Host(id)
+	if err != nil {
+		return errors.Wrap(err, "find host for refetch")
+	}
+
+	host.RefetchRequested = true
+	if err := svc.ds.SaveHost(host); err != nil {
+		return errors.Wrap(err, "save host")
+	}
+
+	return nil
+}

server/service/service_hosts_test.go

@@ -100,3 +100,21 @@ func TestHostDetails(t *testing.T) {
 	assert.Equal(t, expectedLabels, hostDetail.Labels)
 	assert.Equal(t, expectedPacks, hostDetail.Packs)
 }
+
+func TestRefetchHost(t *testing.T) {
+	ds := new(mock.Store)
+	svc := service{ds: ds}
+
+	host := &kolide.Host{ID: 3}
+	ctx := context.Background()
+
+	ds.HostFunc = func(hid uint) (*kolide.Host, error) {
+		return host, nil
+	}
+	ds.SaveHostFunc = func(host *kolide.Host) error {
+		assert.True(t, host.RefetchRequested)
+		return nil
+	}
+
+	require.NoError(t, svc.RefetchHost(ctx, host.ID))
+}

server/service/service_osquery.go

@@ -861,7 +861,7 @@ func ingestSoftware(logger log.Logger, host *kolide.Host, rows []map[string]stri
 // osqueryd to fill in the host details
 func (svc service) hostDetailQueries(host kolide.Host) (map[string]string, error) {
 	queries := make(map[string]string)
-	if host.DetailUpdateTime.After(svc.clock.Now().Add(-svc.config.Osquery.DetailUpdateInterval)) {
+	if host.DetailUpdateTime.After(svc.clock.Now().Add(-svc.config.Osquery.DetailUpdateInterval)) && !host.RefetchRequested {
 		// No need to update already fresh details
 		return queries, nil
 	}
@@ -959,6 +959,9 @@ func (svc service) ingestDetailQuery(host *kolide.Host, name string, rows []map[
 		}
 	}
 
+	// Refetch is no longer needed after ingesting details.
+	host.RefetchRequested = false
+
 	return nil
 }
 

server/service/service_osquery_test.go

@@ -274,7 +274,14 @@ func TestHostDetailQueries(t *testing.T) {
 
 	queries, err := svc.hostDetailQueries(host)
 	assert.Nil(t, err)
-	assert.Empty(t, queries, 0)
+	assert.Empty(t, queries)
+
+	// With refetch requested queries should be returned
+	host.RefetchRequested = true
+	queries, err = svc.hostDetailQueries(host)
+	assert.Nil(t, err)
+	assert.NotEmpty(t, queries)
+	host.RefetchRequested = false
 
 	// Advance the time
 	mockClock.AddTime(1*time.Hour + 1*time.Minute)

server/service/transport_hosts.go

@@ -29,6 +29,14 @@ func decodeDeleteHostRequest(ctx context.Context, r *http.Request) (interface{},
 	return deleteHostRequest{ID: id}, nil
 }
 
+func decodeRefetchHostRequest(ctx context.Context, r *http.Request) (interface{}, error) {
+	id, err := idFromRequest(r, "id")
+	if err != nil {
+		return nil, err
+	}
+	return refetchHostRequest{ID: id}, nil
+}
+
 func decodeListHostsRequest(ctx context.Context, r *http.Request) (interface{}, error) {
 	hopt, err := hostListOptionsFromRequest(r)
 	if err != nil {