Osqueryd Service (#119)

* host service stuff

* working on the osquery service api

* log status and results logs

datastore/datastore_test.go

@@ -90,7 +90,7 @@ var enrollTests = []struct {
 	},
 }
 
-func testEnrollHost(t *testing.T, db kolide.OsqueryStore) {
+func testEnrollHost(t *testing.T, db kolide.HostStore) {
 	var hosts []*kolide.Host
 	for i, tt := range enrollTests {
 		h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.ip, tt.platform, tt.nodeKeySize)
@@ -145,7 +145,7 @@ func testEnrollHost(t *testing.T, db kolide.OsqueryStore) {
 
 }
 
-func testAuthenticateHost(t *testing.T, db kolide.OsqueryStore) {
+func testAuthenticateHost(t *testing.T, db kolide.HostStore) {
 	for i, tt := range enrollTests {
 		h, err := db.EnrollHost(tt.uuid, tt.hostname, tt.ip, tt.platform, tt.nodeKeySize)
 		if err != nil {

datastore/gorm.go

@@ -175,6 +175,44 @@ func (orm gormDB) SaveHost(host *kolide.Host) error {
 	return nil
 }
 
+func (orm gormDB) DeleteHost(host *kolide.Host) error {
+	return orm.DB.Delete(host).Error
+}
+
+func (orm gormDB) Host(id uint) (*kolide.Host, error) {
+	host := &kolide.Host{
+		ID: id,
+	}
+	err := orm.DB.Where(host).First(host).Error
+	if err != nil {
+		return nil, err
+	}
+	return host, nil
+}
+
+func (orm gormDB) Hosts() ([]*kolide.Host, error) {
+	var hosts []*kolide.Host
+	err := orm.DB.Find(&hosts).Error
+	if err != nil {
+		return nil, err
+	}
+	return hosts, nil
+}
+
+func (orm gormDB) NewHost(host *kolide.Host) (*kolide.Host, error) {
+	if host == nil {
+		return nil, errors.New(
+			"error creating host",
+			"nil pointer passed to NewHost",
+		)
+	}
+	err := orm.DB.Create(host).Error
+	if err != nil {
+		return nil, err
+	}
+	return host, err
+}
+
 func (orm gormDB) MarkHostSeen(host *kolide.Host, t time.Time) error {
 	updateTime := time.Now()
 	err := orm.DB.Exec("UPDATE hosts SET updated_at=? WHERE node_key=?", t, host.NodeKey).Error
@@ -476,13 +514,13 @@ SELECT
   q.differential,
   q.platform,
   q.version
-FROM 
-  queries q 
-JOIN 
-  pack_queries pq 
-ON 
-  pq.query_id = q.id 
-AND 
+FROM
+  queries q
+JOIN
+  pack_queries pq
+ON
+  pq.query_id = q.id
+AND
   pq.pack_id = ?;
 `, pack.ID).Rows()
 	if err != nil && err != gorm.ErrRecordNotFound {

kolide/datastore.go

@@ -6,6 +6,7 @@ type Datastore interface {
 	QueryStore
 	PackStore
 	OsqueryStore
+	HostStore
 	EmailStore
 	SessionStore
 	Name() string

kolide/hosts.go

@@ -0,0 +1,45 @@
+package kolide
+
+import (
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+type HostStore interface {
+	NewHost(host *Host) (*Host, error)
+	SaveHost(host *Host) error
+	DeleteHost(host *Host) error
+	Host(id uint) (*Host, error)
+	Hosts() ([]*Host, error)
+	EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*Host, error)
+	AuthenticateHost(nodeKey string) (*Host, error)
+	MarkHostSeen(host *Host, t time.Time) error
+}
+
+type HostService interface {
+	GetAllHosts(ctx context.Context) ([]*Host, error)
+	GetHost(ctx context.Context, id uint) (*Host, error)
+	NewHost(ctx context.Context, p HostPayload) (*Host, error)
+	ModifyHost(ctx context.Context, id uint, p HostPayload) (*Host, error)
+	DeleteHost(ctx context.Context, id uint) error
+}
+
+type HostPayload struct {
+	NodeKey   *string
+	HostName  *string
+	UUID      *string
+	IPAddress *string
+	Platform  *string
+}
+
+type Host struct {
+	ID        uint `gorm:"primary_key"`
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	NodeKey   string `gorm:"unique_index:idx_host_unique_nodekey"`
+	HostName  string
+	UUID      string `gorm:"unique_index:idx_host_unique_uuid"`
+	IPAddress string
+	Platform  string
+}

kolide/osquery.go

@@ -8,11 +8,6 @@ import (
 )
 
 type OsqueryStore interface {
-	EnrollHost(uuid, hostname, ip, platform string, nodeKeySize int) (*Host, error)
-	AuthenticateHost(nodeKey string) (*Host, error)
-	SaveHost(host *Host) error
-	MarkHostSeen(host *Host, t time.Time) error
-
 	LabelQueriesForHost(host *Host, cutoff time.Time) (map[string]string, error)
 	RecordLabelQueryExecutions(host *Host, results map[string]bool, t time.Time) error
 	NewLabel(label *Label) error
@@ -20,30 +15,39 @@ type OsqueryStore interface {
 
 type OsqueryService interface {
 	EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier string) (string, error)
-	GetClientConfig(ctx context.Context, action string, data *json.RawMessage) (*OsqueryConfig, error)
-	Log(ctx context.Context, logType string, data *json.RawMessage) error
+	GetClientConfig(ctx context.Context, action string, data json.RawMessage) (OsqueryConfig, error)
 	GetDistributedQueries(ctx context.Context) (map[string]string, error)
-	LogDistributedQueryResults(ctx context.Context, queries map[string][]map[string]string) error
+	SubmitDistributedQueryResults(ctx context.Context, results OsqueryDistributedQueryResults) error
+	SubmitStatusLogs(ctx context.Context, logs []OsqueryResultLog) error
+	SubmitResultsLogs(ctx context.Context, logs []OsqueryStatusLog) error
 }
 
-type Host struct {
-	ID        uint `gorm:"primary_key"`
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	NodeKey   string `gorm:"unique_index:idx_host_unique_nodekey"`
-	HostName  string
-	UUID      string `gorm:"unique_index:idx_host_unique_uuid"`
-	IPAddress string
-	Platform  string
-}
+type OsqueryDistributedQueryResults map[string][]map[string]string
 
 type OsqueryConfig struct {
 	Packs    []Pack
 	Schedule []Query
-	Options  map[string]interface{}
 }
 
-// TODO: move this to just use OsqueryServerStore.LabelQueriesForHot
+type OsqueryResultLog struct {
+	Name           string            `json:"name"`
+	HostIdentifier string            `json:"hostIdentifier"`
+	UnixTime       string            `json:"unixTime"`
+	CalendarTime   string            `json:"calendarTime"`
+	Columns        map[string]string `json:"columns"`
+	Action         string            `json:"action"`
+}
+
+type OsqueryStatusLog struct {
+	Severity    string            `json:"severity"`
+	Filename    string            `json:"filename"`
+	Line        string            `json:"line"`
+	Message     string            `json:"message"`
+	Version     string            `json:"version"`
+	Decorations map[string]string `json:"decorations"`
+}
+
+// TODO: move this to just use LabelQueriesForHot
 // LabelQueriesForHost calculates the appropriate update cutoff (given
 // interval) and uses the datastore to retrieve the label queries for the
 // provided host.

kolide/service.go

@@ -7,4 +7,5 @@ type Service interface {
 	PackService
 	QueryService
 	OsqueryService
+	HostService
 }

server/service_hosts.go

@@ -0,0 +1,92 @@
+package server
+
+import (
+	"github.com/kolide/kolide-ose/kolide"
+	"golang.org/x/net/context"
+)
+
+func (svc service) GetAllHosts(ctx context.Context) ([]*kolide.Host, error) {
+	return svc.ds.Hosts()
+}
+
+func (svc service) GetHost(ctx context.Context, id uint) (*kolide.Host, error) {
+	return svc.ds.Host(id)
+}
+
+func (svc service) NewHost(ctx context.Context, p kolide.HostPayload) (*kolide.Host, error) {
+	var host kolide.Host
+
+	if p.HostName != nil {
+		host.HostName = *p.HostName
+	}
+
+	if p.IPAddress != nil {
+		host.IPAddress = *p.IPAddress
+	}
+
+	if p.NodeKey != nil {
+		host.NodeKey = *p.NodeKey
+	}
+
+	if p.Platform != nil {
+		host.Platform = *p.Platform
+	}
+
+	if p.UUID != nil {
+		host.UUID = *p.UUID
+	}
+
+	_, err := svc.ds.NewHost(&host)
+	if err != nil {
+		return nil, err
+	}
+	return &host, nil
+}
+
+func (svc service) ModifyHost(ctx context.Context, id uint, p kolide.HostPayload) (*kolide.Host, error) {
+	host, err := svc.ds.Host(id)
+	if err != nil {
+		return nil, err
+	}
+
+	if p.HostName != nil {
+		host.HostName = *p.HostName
+	}
+
+	if p.IPAddress != nil {
+		host.IPAddress = *p.IPAddress
+	}
+
+	if p.NodeKey != nil {
+		host.NodeKey = *p.NodeKey
+	}
+
+	if p.Platform != nil {
+		host.Platform = *p.Platform
+	}
+
+	if p.UUID != nil {
+		host.UUID = *p.UUID
+	}
+
+	err = svc.ds.SaveHost(host)
+	if err != nil {
+		return nil, err
+	}
+
+	return host, nil
+}
+
+func (svc service) DeleteHost(ctx context.Context, id uint) error {
+	host, err := svc.ds.Host(id)
+	if err != nil {
+		return err
+	}
+
+	err = svc.ds.DeleteHost(host)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

server/service_hosts_test.go

@@ -0,0 +1,124 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/kolide/kolide-ose/datastore"
+	"github.com/kolide/kolide-ose/kolide"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+)
+
+func TestGetAllHosts(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	svc, err := NewService(testConfig(ds))
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	hosts, err := svc.GetAllHosts(ctx)
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 0)
+
+	_, err = ds.NewHost(&kolide.Host{
+		HostName: "foo",
+	})
+	assert.Nil(t, err)
+
+	hosts, err = svc.GetAllHosts(ctx)
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 1)
+}
+
+func TestGetHost(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	svc, err := NewService(testConfig(ds))
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	host, err := ds.NewHost(&kolide.Host{
+		HostName: "foo",
+	})
+	assert.Nil(t, err)
+	assert.NotZero(t, host.ID)
+
+	hostVerify, err := svc.GetHost(ctx, host.ID)
+	assert.Nil(t, err)
+
+	assert.Equal(t, host.ID, hostVerify.ID)
+}
+
+func TestNewHost(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	svc, err := NewService(testConfig(ds))
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	hostname := "foo"
+	_, err = svc.NewHost(ctx, kolide.HostPayload{
+		HostName: &hostname,
+	})
+
+	assert.Nil(t, err)
+
+	hosts, err := ds.Hosts()
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 1)
+}
+
+func TestModifyHost(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	svc, err := NewService(testConfig(ds))
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	host, err := ds.NewHost(&kolide.Host{
+		HostName: "foo",
+	})
+	assert.Nil(t, err)
+	assert.NotZero(t, host.ID)
+
+	newHostname := "bar"
+	hostVerify, err := svc.ModifyHost(ctx, host.ID, kolide.HostPayload{
+		HostName: &newHostname,
+	})
+	assert.Nil(t, err)
+
+	assert.Equal(t, host.ID, hostVerify.ID)
+	assert.Equal(t, "bar", hostVerify.HostName)
+}
+
+func TestDeleteHost(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	svc, err := NewService(testConfig(ds))
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	host, err := ds.NewHost(&kolide.Host{
+		HostName: "foo",
+	})
+	assert.Nil(t, err)
+	assert.NotZero(t, host.ID)
+
+	err = svc.DeleteHost(ctx, host.ID)
+	assert.Nil(t, err)
+
+	hosts, err := ds.Hosts()
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 0)
+
+}

server/service_osquery.go

@@ -3,6 +3,7 @@ package server
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 
 	"github.com/kolide/kolide-ose/errors"
 	"github.com/kolide/kolide-ose/kolide"
@@ -25,11 +26,28 @@ func (svc service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier
 	return host.NodeKey, nil
 }
 
-func (svc service) GetClientConfig(ctx context.Context, action string, data *json.RawMessage) (*kolide.OsqueryConfig, error) {
-	return nil, nil
+func (svc service) GetClientConfig(ctx context.Context, action string, data json.RawMessage) (kolide.OsqueryConfig, error) {
+	var config kolide.OsqueryConfig
+	return config, nil
 }
 
-func (svc service) Log(ctx context.Context, logType string, data *json.RawMessage) error {
+func (svc service) SubmitStatusLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error {
+	for _, log := range logs {
+		err := json.NewEncoder(svc.osqueryStatusLogWriter).Encode(log)
+		if err != nil {
+			return errors.NewFromError(err, http.StatusInternalServerError, "error writing status log")
+		}
+	}
+	return nil
+}
+
+func (svc service) SubmitResultsLogs(ctx context.Context, logs []kolide.OsqueryStatusLog) error {
+	for _, log := range logs {
+		err := json.NewEncoder(svc.osqueryResultsLogWriter).Encode(log)
+		if err != nil {
+			return errors.NewFromError(err, http.StatusInternalServerError, "error writing result log")
+		}
+	}
 	return nil
 }
 
@@ -41,6 +59,6 @@ func (svc service) GetDistributedQueries(ctx context.Context) (map[string]string
 	return queries, nil
 }
 
-func (svc service) LogDistributedQueryResults(ctx context.Context, queries map[string][]map[string]string) error {
+func (svc service) SubmitDistributedQueryResults(ctx context.Context, results kolide.OsqueryDistributedQueryResults) error {
 	return nil
 }

server/service_osquery_test.go

@@ -0,0 +1,72 @@
+package server
+
+import (
+	"context"
+	"testing"
+
+	kitlog "github.com/go-kit/kit/log"
+	"github.com/kolide/kolide-ose/datastore"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEnrollAgent(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	config := ServiceConfig{
+		Datastore:           ds,
+		Logger:              kitlog.NewNopLogger(),
+		BcryptCost:          12,
+		SaltKeySize:         24,
+		SessionCookieName:   "KolideSession",
+		OsqueryNodeKeySize:  12,
+		OsqueryEnrollSecret: "foobar",
+	}
+	svc, err := NewService(config)
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	hosts, err := ds.Hosts()
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 0)
+
+	nodeKey, err := svc.EnrollAgent(ctx, "foobar", "host123")
+	assert.Nil(t, err)
+	assert.NotEmpty(t, nodeKey)
+
+	hosts, err = ds.Hosts()
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 1)
+}
+
+func TestEnrollAgentIncorrectEnrollSecret(t *testing.T) {
+	ds, err := datastore.New("gorm-sqlite3", ":memory:")
+	assert.Nil(t, err)
+
+	config := ServiceConfig{
+		Datastore:           ds,
+		Logger:              kitlog.NewNopLogger(),
+		BcryptCost:          12,
+		SaltKeySize:         24,
+		SessionCookieName:   "KolideSession",
+		OsqueryNodeKeySize:  12,
+		OsqueryEnrollSecret: "foobar",
+	}
+	svc, err := NewService(config)
+	assert.Nil(t, err)
+
+	ctx := context.Background()
+
+	hosts, err := ds.Hosts()
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 0)
+
+	nodeKey, err := svc.EnrollAgent(ctx, "not_correct", "host123")
+	assert.NotNil(t, err)
+	assert.Empty(t, nodeKey)
+
+	hosts, err = ds.Hosts()
+	assert.Nil(t, err)
+	assert.Len(t, hosts, 0)
+}