Send configuration + label queries in distributed reads (#215)

This PR is the beginning of distributed query work. For now we are focusing on using the distributed query subsystem to retrieve the basic configuration information (currently just platform), and run the label queries.

A mockable clock interface is also added to the service struct, allowing us to inject a clock as a dependency, and write unit tests that can control the time.

cli/prepare.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 
+	"github.com/WatchBeam/clock"
 	kitlog "github.com/go-kit/kit/log"
 	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
@@ -82,7 +83,7 @@ To setup kolide infrastructure, use one of the available commands.
 				Enabled:  &enabled,
 				Admin:    &isAdmin,
 			}
-			svc, err := server.NewService(ds, kitlog.NewNopLogger(), config, nil)
+			svc, err := server.NewService(ds, kitlog.NewNopLogger(), config, nil, clock.C)
 			if err != nil {
 				initFatal(err, "creating service")
 			}

cli/serve.go

@@ -8,6 +8,7 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/WatchBeam/clock"
 	kitlog "github.com/go-kit/kit/log"
 	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
@@ -73,7 +74,7 @@ the way that the kolide server works.
 				}
 			}
 
-			svc, err := server.NewService(ds, logger, config, mailService)
+			svc, err := server.NewService(ds, logger, config, mailService, clock.C)
 			if err != nil {
 				initFatal(err, "initializing service")
 			}

config/config.go

@@ -58,10 +58,11 @@ type SessionConfig struct {
 
 // OsqueryConfig defines configs related to osquery
 type OsqueryConfig struct {
-	EnrollSecret  string
-	NodeKeySize   int
-	StatusLogFile string
-	ResultLogFile string
+	EnrollSecret        string
+	NodeKeySize         int
+	StatusLogFile       string
+	ResultLogFile       string
+	LabelUpdateInterval time.Duration
 }
 
 // LoggingConfig defines configs related to logging
@@ -123,6 +124,7 @@ func (man Manager) addConfigs() {
 	man.addConfigInt("osquery.node_key_size", 24)
 	man.addConfigString("osquery.status_log_file", "/tmp/osquery_status")
 	man.addConfigString("osquery.result_log_file", "/tmp/osquery_result")
+	man.addConfigDuration("osquery.label_update_interval", 1*time.Hour)
 
 	// Logging
 	man.addConfigBool("logging.debug", false)
@@ -166,10 +168,11 @@ func (man Manager) LoadConfig() KolideConfig {
 			Duration: man.getConfigDuration("session.duration"),
 		},
 		Osquery: OsqueryConfig{
-			EnrollSecret:  man.getConfigString("osquery.enroll_secret"),
-			NodeKeySize:   man.getConfigInt("osquery.node_key_size"),
-			StatusLogFile: man.getConfigString("osquery.status_log_file"),
-			ResultLogFile: man.getConfigString("osquery.result_log_file"),
+			EnrollSecret:        man.getConfigString("osquery.enroll_secret"),
+			NodeKeySize:         man.getConfigInt("osquery.node_key_size"),
+			StatusLogFile:       man.getConfigString("osquery.status_log_file"),
+			ResultLogFile:       man.getConfigString("osquery.result_log_file"),
+			LabelUpdateInterval: man.getConfigDuration("osquery.label_update_interval"),
 		},
 		Logging: LoggingConfig{
 			Debug:         man.getConfigBool("logging.debug"),
@@ -357,10 +360,11 @@ func TestConfig() KolideConfig {
 			Duration: 24 * 90 * time.Hour,
 		},
 		Osquery: OsqueryConfig{
-			EnrollSecret:  "",
-			NodeKeySize:   24,
-			StatusLogFile: "",
-			ResultLogFile: "",
+			EnrollSecret:        "",
+			NodeKeySize:         24,
+			StatusLogFile:       "",
+			ResultLogFile:       "",
+			LabelUpdateInterval: 1 * time.Hour,
 		},
 		Logging: LoggingConfig{
 			Debug:         true,

datastore/gorm.go

@@ -374,7 +374,7 @@ matches = VALUES(matches)
 `
 	}
 
-	if err := orm.DB.Debug().Exec(queryString, vals...).Error; err != nil {
+	if err := orm.DB.Exec(queryString, vals...).Error; err != nil {
 		return errors.DatabaseError(err)
 	}
 

glide.lock

@@ -1,5 +1,5 @@
-hash: 082be2432c000297d8d76bd1dc0f2be4e98954d0f49682603469e1b3056790f5
-updated: 2016-08-29T09:08:22.504560027-04:00
+hash: 3bf651074168b7611aabf854ebba43d0a41dd3cb3fbd0892268f9163e4240f9f
+updated: 2016-09-20T11:30:44.994118865-07:00
 imports:
 - name: github.com/alecthomas/template
   version: a0175ee3bccc567396460bf5acd36800cb10c49c
@@ -13,6 +13,8 @@ imports:
   - spew
 - name: github.com/dgrijalva/jwt-go
   version: 01aeca54ebda6e0fbfafd0a524d234159c05ec20
+  subpackages:
+  - request
 - name: github.com/elazarl/go-bindata-assetfs
   version: 9a6736ed45b44bf3835afeebb3034b57ed329f3e
 - name: github.com/fsnotify/fsnotify
@@ -42,8 +44,6 @@ imports:
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/golang/mock
   version: bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
-  subpackages:
-  - gomock
 - name: github.com/golang/protobuf
   version: c3cefd437628a0b7d31b34fe44b3a7a540e98527
   subpackages:
@@ -119,6 +119,9 @@ imports:
   version: d77da356e56a7428ad25149ca77381849a6a5232
   subpackages:
   - assert
+  - require
+- name: github.com/WatchBeam/clock
+  version: d5a6612f3d9a070b964adc65dbd94c6080458f83
 - name: golang.org/x/crypto
   version: bc89c496413265e715159bdc8478ee9a92fdc265
   subpackages:

glide.yaml

@@ -78,3 +78,4 @@ import:
 - package: gopkg.in/natefinch/lumberjack.v2
   version: v2.0
 - package: github.com/golang/mock
+- package: github.com/WatchBeam/clock

server/handler_test.go

@@ -5,9 +5,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	kitlog "github.com/go-kit/kit/log"
 	"github.com/gorilla/mux"
-	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/net/context"
@@ -17,7 +15,7 @@ func TestAPIRoutes(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()

server/http_auth.go

@@ -129,6 +129,14 @@ func emptyVC() *viewerContext {
 	return &viewerContext{}
 }
 
+func osqueryHostFromContext(ctx context.Context) (*kolide.Host, error) {
+	host, ok := ctx.Value("osqueryHost").(*kolide.Host)
+	if !ok {
+		return nil, errNoContext
+	}
+	return host, nil
+}
+
 func viewerContextFromContext(ctx context.Context) (*viewerContext, error) {
 	vc, ok := ctx.Value("viewerContext").(*viewerContext)
 	if !ok {

server/http_auth_test.go

@@ -23,7 +23,7 @@ import (
 
 func TestLogin(t *testing.T) {
 	ds, _ := datastore.New("inmem", "")
-	svc, _ := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, _ := newTestService(ds)
 	createTestUsers(t, ds)
 	logger := kitlog.NewLogfmtLogger(os.Stdout)
 

server/service.go

@@ -5,25 +5,15 @@ package server
 import (
 	"io"
 
+	"github.com/WatchBeam/clock"
 	kitlog "github.com/go-kit/kit/log"
 	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/kolide"
 	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 )
 
-// configuration defaults
-// TODO move to main?
-const (
-	defaultBcryptCost       int    = 12
-	defaultSaltKeySize      int    = 24
-	defaultCookieName       string = "KolideSession"
-	defaultEnrollSecret     string = "xxx change me"
-	defaultNodeKeySize      int    = 24
-	defaultSMTPTokenKeySize int    = 24
-)
-
 // NewService creates a new service from the config struct
-func NewService(ds kolide.Datastore, logger kitlog.Logger, kolideConfig config.KolideConfig, mailService kolide.MailService) (kolide.Service, error) {
+func NewService(ds kolide.Datastore, logger kitlog.Logger, kolideConfig config.KolideConfig, mailService kolide.MailService, c clock.Clock) (kolide.Service, error) {
 	var svc kolide.Service
 
 	logFile := func(path string) io.Writer {
@@ -39,6 +29,7 @@ func NewService(ds kolide.Datastore, logger kitlog.Logger, kolideConfig config.K
 		ds:     ds,
 		logger: logger,
 		config: kolideConfig,
+		clock:  c,
 
 		osqueryStatusLogWriter:  logFile(kolideConfig.Osquery.StatusLogFile),
 		osqueryResultsLogWriter: logFile(kolideConfig.Osquery.ResultLogFile),
@@ -52,6 +43,7 @@ type service struct {
 	ds     kolide.Datastore
 	logger kitlog.Logger
 	config config.KolideConfig
+	clock  clock.Clock
 
 	osqueryStatusLogWriter  io.Writer
 	osqueryResultsLogWriter io.Writer

server/service_hosts_test.go

@@ -3,8 +3,6 @@ package server
 import (
 	"testing"
 
-	kitlog "github.com/go-kit/kit/log"
-	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
 	"github.com/kolide/kolide-ose/kolide"
 	"github.com/stretchr/testify/assert"
@@ -15,7 +13,7 @@ func TestGetAllHosts(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -38,7 +36,7 @@ func TestGetHost(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -59,7 +57,7 @@ func TestNewHost(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -80,7 +78,7 @@ func TestModifyHost(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -105,7 +103,7 @@ func TestDeleteHost(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()

server/service_osquery.go

@@ -10,6 +10,14 @@ import (
 	"golang.org/x/net/context"
 )
 
+type osqueryError struct {
+	message string
+}
+
+func (e osqueryError) Error() string {
+	return e.message
+}
+
 func (svc service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier string) (string, error) {
 	if enrollSecret != svc.config.Osquery.EnrollSecret {
 		return "", errors.New(
@@ -51,10 +59,52 @@ func (svc service) SubmitResultsLogs(ctx context.Context, logs []kolide.OsqueryS
 	return nil
 }
 
-func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string, error) {
-	var queries map[string]string
+// hostLabelQueryPrefix is appended before the query name when a query is
+// provided as a label query. This allows the results to be retrieved when
+// osqueryd writes the distributed query results.
+const hostLabelQueryPrefix = "kolide_label_query_"
 
-	queries["id1"] = "select * from osquery_info"
+// hostDetailQueryPrefix is appended before the query name when a query is
+// provided as a detail query.
+const hostDetailQueryPrefix = "kolide_detail_query_"
+
+// hostDetailQueries returns the map of queries that should be executed by
+// osqueryd to fill in the host details
+func hostDetailQueries(host kolide.Host) map[string]string {
+	queries := make(map[string]string)
+	if host.Platform == "" {
+		queries[hostDetailQueryPrefix+"platform"] = "select build_platform from osquery_info;"
+	}
+	return queries
+}
+
+func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string, error) {
+	queries := make(map[string]string)
+
+	host, err := osqueryHostFromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	queries = hostDetailQueries(*host)
+	if len(queries) > 0 {
+		// If the host details need to be updated, we should do so
+		// before checking for any other queries
+		return queries, nil
+	}
+
+	// Retrieve the label queries that should be updated
+	cutoff := svc.clock.Now().Add(-svc.config.Osquery.LabelUpdateInterval)
+	labelQueries, err := svc.ds.LabelQueriesForHost(host, cutoff)
+	if err != nil {
+		return nil, err
+	}
+
+	for name, query := range labelQueries {
+		queries[hostLabelQueryPrefix+name] = query
+	}
+
+	// TODO: retrieve the active distributed queries for this host
 
 	return queries, nil
 }

server/service_osquery_test.go

@@ -2,21 +2,22 @@ package server
 
 import (
 	"context"
+	"fmt"
 	"testing"
+	"time"
 
-	kitlog "github.com/go-kit/kit/log"
-	"github.com/kolide/kolide-ose/config"
+	"github.com/WatchBeam/clock"
 	"github.com/kolide/kolide-ose/datastore"
+	"github.com/kolide/kolide-ose/kolide"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestEnrollAgent(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	conf := config.TestConfig()
-	conf.Osquery.EnrollSecret = "foobar"
-	svc, err := NewService(ds, kitlog.NewNopLogger(), conf, nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -25,7 +26,7 @@ func TestEnrollAgent(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Len(t, hosts, 0)
 
-	nodeKey, err := svc.EnrollAgent(ctx, "foobar", "host123")
+	nodeKey, err := svc.EnrollAgent(ctx, "", "host123")
 	assert.Nil(t, err)
 	assert.NotEmpty(t, nodeKey)
 
@@ -38,9 +39,7 @@ func TestEnrollAgentIncorrectEnrollSecret(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	conf := config.TestConfig()
-	conf.Osquery.EnrollSecret = "foobar"
-	svc, err := NewService(ds, kitlog.NewNopLogger(), conf, nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -57,3 +56,174 @@ func TestEnrollAgentIncorrectEnrollSecret(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Len(t, hosts, 0)
 }
+
+func TestHostDetailQueries(t *testing.T) {
+	host := kolide.Host{
+		ID:        1,
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+		NodeKey:   "test_key",
+		HostName:  "test_hostname",
+		UUID:      "test_uuid",
+	}
+
+	queries := hostDetailQueries(host)
+	assert.Len(t, queries, 1)
+	if assert.Contains(t, queries, "kolide_detail_query_platform") {
+		assert.Equal(t,
+			"select build_platform from osquery_info;",
+			queries["kolide_detail_query_platform"],
+		)
+	}
+
+	host.Platform = "test_platform"
+
+	queries = hostDetailQueries(host)
+	assert.Len(t, queries, 0)
+}
+
+func TestGetDistributedQueries(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	mockClock := clock.NewMockClock()
+
+	svc, err := newTestServiceWithClock(ds, mockClock)
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	_, err = svc.EnrollAgent(ctx, "", "host123")
+	assert.Nil(t, err)
+
+	hosts, err := ds.Hosts()
+	require.Nil(t, err)
+	require.Len(t, hosts, 1)
+	host := hosts[0]
+
+	ctx = context.WithValue(ctx, "osqueryHost", host)
+
+	// With no platform set, we should get the details query
+	queries, err := svc.GetDistributedQueries(ctx)
+	assert.Nil(t, err)
+	assert.Len(t, queries, 1)
+	if assert.Contains(t, queries, "kolide_detail_query_platform") {
+		assert.Equal(t,
+			"select build_platform from osquery_info;",
+			queries["kolide_detail_query_platform"],
+		)
+	}
+
+	host.Platform = "darwin"
+
+	ctx = context.WithValue(ctx, "osqueryHost", host)
+
+	// With the platform set, we should get the label queries (but none
+	// exist yet)
+	queries, err = svc.GetDistributedQueries(ctx)
+	assert.Nil(t, err)
+	assert.Len(t, queries, 0)
+
+	// Add some queries and labels to ensure they are returned
+
+	labelQueries := []*kolide.Query{
+		&kolide.Query{
+			ID:       1,
+			Name:     "query1",
+			Platform: "darwin",
+			Query:    "query1",
+		},
+		&kolide.Query{
+			ID:       2,
+			Name:     "query2",
+			Platform: "darwin",
+			Query:    "query2",
+		},
+		&kolide.Query{
+			ID:       3,
+			Name:     "query3",
+			Platform: "darwin",
+			Query:    "query3",
+		},
+	}
+
+	expectQueries := make(map[string]string)
+
+	for _, query := range labelQueries {
+		assert.NoError(t, ds.NewQuery(query))
+		expectQueries[fmt.Sprintf("kolide_label_query_%d", query.ID)] = query.Query
+	}
+	// this one should not show up
+	assert.NoError(t, ds.NewQuery(&kolide.Query{
+		ID:       4,
+		Name:     "query4",
+		Platform: "not_darwin",
+		Query:    "query4",
+	}))
+
+	labels := []*kolide.Label{
+		&kolide.Label{
+			Name:    "label1",
+			QueryID: 1,
+		},
+		&kolide.Label{
+			Name:    "label2",
+			QueryID: 2,
+		},
+		&kolide.Label{
+			Name:    "label3",
+			QueryID: 3,
+		},
+		&kolide.Label{
+			Name:    "label4",
+			QueryID: 4,
+		},
+	}
+
+	for _, label := range labels {
+		assert.NoError(t, ds.NewLabel(label))
+	}
+
+	// Now we should get the label queries
+	queries, err = svc.GetDistributedQueries(ctx)
+	assert.Nil(t, err)
+	assert.Len(t, queries, 3)
+	assert.Equal(t, expectQueries, queries)
+
+	// Record a query execution
+	err = ds.RecordLabelQueryExecutions(host, map[string]bool{"1": true}, mockClock.Now())
+	assert.NoError(t, err)
+
+	// Now that query should not be returned
+	queries, err = svc.GetDistributedQueries(ctx)
+	assert.Nil(t, err)
+	assert.Len(t, queries, 2)
+	assert.NotContains(t, queries, "kolide_label_query_1")
+
+	// Advance the time
+	mockClock.AddTime(1*time.Hour + 1*time.Minute)
+
+	// Now we should get all the label queries again
+	queries, err = svc.GetDistributedQueries(ctx)
+	assert.Nil(t, err)
+	assert.Len(t, queries, 3)
+	assert.Equal(t, expectQueries, queries)
+
+	// Record an old query execution -- Shouldn't change the return
+	err = ds.RecordLabelQueryExecutions(host, map[string]bool{"2": true}, mockClock.Now().Add(-10*time.Hour))
+	assert.NoError(t, err)
+	queries, err = svc.GetDistributedQueries(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, expectQueries, queries)
+
+	// Record a newer execution for that query and another
+	err = ds.RecordLabelQueryExecutions(host, map[string]bool{"2": true, "3": false}, mockClock.Now().Add(-1*time.Minute))
+	assert.NoError(t, err)
+
+	// Now these should no longer show up in the necessary to run queries
+	delete(expectQueries, "kolide_label_query_2")
+	delete(expectQueries, "kolide_label_query_3")
+	queries, err = svc.GetDistributedQueries(ctx)
+	assert.NoError(t, err)
+	assert.Equal(t, expectQueries, queries)
+}

server/service_packs_test.go

@@ -3,8 +3,6 @@ package server
 import (
 	"testing"
 
-	kitlog "github.com/go-kit/kit/log"
-	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
 	"github.com/kolide/kolide-ose/kolide"
 	"github.com/stretchr/testify/assert"
@@ -15,7 +13,7 @@ func TestGetAllPacks(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -38,7 +36,7 @@ func TestGetPack(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -60,7 +58,7 @@ func TestNewPack(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -81,7 +79,7 @@ func TestModifyPack(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -107,7 +105,7 @@ func TestDeletePack(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -132,7 +130,7 @@ func TestAddQueryToPack(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -168,7 +166,7 @@ func TestGetQueriesInPack(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -200,7 +198,7 @@ func TestRemoveQueryFromPack(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()

server/service_queries_test.go

@@ -3,8 +3,6 @@ package server
 import (
 	"testing"
 
-	kitlog "github.com/go-kit/kit/log"
-	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
 	"github.com/kolide/kolide-ose/kolide"
 	"github.com/stretchr/testify/assert"
@@ -15,7 +13,7 @@ func TestGetAllQueries(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -39,7 +37,7 @@ func TestGetQuery(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -62,7 +60,7 @@ func TestNewQuery(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -85,7 +83,7 @@ func TestModifyQuery(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()
@@ -112,7 +110,7 @@ func TestDeleteQuery(t *testing.T) {
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 
 	ctx := context.Background()

server/service_sessions_test.go

@@ -4,8 +4,6 @@ import (
 	"testing"
 	"time"
 
-	kitlog "github.com/go-kit/kit/log"
-	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
 	"github.com/kolide/kolide-ose/kolide"
 	"github.com/stretchr/testify/assert"
@@ -63,7 +61,7 @@ func setupLoginTests(t *testing.T) (kolide.Service, kolide.UserPayload, kolide.U
 	ds, err := datastore.New("gorm-sqlite3", ":memory:")
 	assert.Nil(t, err)
 
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 	payload := kolide.UserPayload{
 		Username: stringPtr("foo"),

server/service_users_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 	"time"
 
-	kitlog "github.com/go-kit/kit/log"
 	"github.com/kolide/kolide-ose/config"
 	"github.com/kolide/kolide-ose/datastore"
 	"github.com/kolide/kolide-ose/kolide"
@@ -19,7 +18,7 @@ func TestAuthenticatedUser(t *testing.T) {
 	ds, err := datastore.New("inmem", "")
 	assert.Nil(t, err)
 	createTestUsers(t, ds)
-	svc, err := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, err := newTestService(ds)
 	assert.Nil(t, err)
 	admin1, err := ds.User("admin1")
 	assert.Nil(t, err)
@@ -114,7 +113,7 @@ func TestRequestPasswordReset(t *testing.T) {
 
 func TestCreateUser(t *testing.T) {
 	ds, _ := datastore.New("inmem", "")
-	svc, _ := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, _ := newTestService(ds)
 	ctx := context.Background()
 
 	var createUserTests = []struct {
@@ -186,7 +185,7 @@ func TestCreateUser(t *testing.T) {
 
 func TestChangeUserPassword(t *testing.T) {
 	ds, _ := datastore.New("inmem", "")
-	svc, _ := NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil)
+	svc, _ := newTestService(ds)
 	createTestUsers(t, ds)
 	var passwordChangeTests = []struct {
 		token       string

server/test_util.go

@@ -0,0 +1,16 @@
+package server
+
+import (
+	"github.com/WatchBeam/clock"
+	kitlog "github.com/go-kit/kit/log"
+	"github.com/kolide/kolide-ose/config"
+	"github.com/kolide/kolide-ose/kolide"
+)
+
+func newTestService(ds kolide.Datastore) (kolide.Service, error) {
+	return NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil, clock.C)
+}
+
+func newTestServiceWithClock(ds kolide.Datastore, c clock.Clock) (kolide.Service, error) {
+	return NewService(ds, kitlog.NewNopLogger(), config.TestConfig(), nil, c)
+}