diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c91fd76a..666e00e67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Kolide Fleet 1.0.6 (TBD, 2017) + +* Fixed bugs that caused logs to sometimes be ommited from the logwriter. (#1636, #1617) + * Fixed a bug where SAML client would create too many HTTP connections. (#1587) * Fixed bug in which default query was run instead of entered query. (#1611) diff --git a/server/kolide/osquery.go b/server/kolide/osquery.go index ec519114e..54a81f8aa 100644 --- a/server/kolide/osquery.go +++ b/server/kolide/osquery.go @@ -2,6 +2,7 @@ package kolide import ( "context" + "encoding/json" ) type OsqueryService interface { @@ -18,7 +19,7 @@ type OsqueryService interface { GetDistributedQueries(ctx context.Context) (queries map[string]string, accelerate uint, err error) SubmitDistributedQueryResults(ctx context.Context, results OsqueryDistributedQueryResults, statuses map[string]string) (err error) SubmitStatusLogs(ctx context.Context, logs []OsqueryStatusLog) (err error) - SubmitResultLogs(ctx context.Context, logs []OsqueryResultLog) (err error) + SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error) } // OsqueryDistributedQueryResults represents the format of the results of an @@ -71,21 +72,6 @@ type OsqueryConfig struct { FilePaths FIMSections `json:"file_paths,omitempty"` } -// OsqueryResultLog is the format of an osquery result log (ie: a differential -// or snapshot query). -type OsqueryResultLog struct { - Name string `json:"name"` - HostIdentifier string `json:"hostIdentifier"` - UnixTime string `json:"unixTime"` - CalendarTime string `json:"calendarTime"` - // Columns stores the columns of differential queries - Columns map[string]string `json:"columns,omitempty"` - // Snapshot stores the rows and columns of snapshot queries - Snapshot []map[string]string `json:"snapshot,omitempty"` - Action string `json:"action"` - Decorations map[string]string `json:"decorations"` -} - // OsqueryStatusLog is the format of an osquery status log. type OsqueryStatusLog struct { Severity string `json:"severity"` diff --git a/server/launcher/launcher.go b/server/launcher/launcher.go index 35274d52d..1ea0f8d77 100644 --- a/server/launcher/launcher.go +++ b/server/launcher/launcher.go @@ -112,13 +112,9 @@ func (svc *launcherWrapper) PublishLogs(ctx context.Context, nodeKey string, log err = svc.tls.SubmitStatusLogs(newCtx, statuses) return "", "", false, errors.Wrap(err, "submit status logs from launcher") case logger.LogTypeSnapshot, logger.LogTypeString: - var results []kolide.OsqueryResultLog + var results []json.RawMessage for _, log := range logs { - var result kolide.OsqueryResultLog - if err := json.Unmarshal([]byte(log), &result); err != nil { - return "", "", false, errors.Wrap(err, "unmarshaling result log") - } - results = append(results, result) + results = append(results, []byte(log)) } err = svc.tls.SubmitResultLogs(newCtx, results) return "", "", false, errors.Wrap(err, "submit result logs from launcher") diff --git a/server/launcher/launcher_test.go b/server/launcher/launcher_test.go index ab2c3b951..cdddf5571 100644 --- a/server/launcher/launcher_test.go +++ b/server/launcher/launcher_test.go @@ -2,6 +2,7 @@ package launcher import ( "context" + "encoding/json" "testing" "github.com/go-kit/kit/log" @@ -148,7 +149,7 @@ func newTLSService(t *testing.T) *mock.TLSService { SubmitStatusLogsFunc: func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) { return }, - SubmitResultLogsFunc: func(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) { + SubmitResultLogsFunc: func(ctx context.Context, logs []json.RawMessage) (err error) { return }, } diff --git a/server/mock/service_osquery.go b/server/mock/service_osquery.go index 50f3c9806..578baccf8 100644 --- a/server/mock/service_osquery.go +++ b/server/mock/service_osquery.go @@ -4,6 +4,7 @@ package mock import ( "context" + "encoding/json" "github.com/kolide/fleet/server/kolide" ) @@ -22,7 +23,7 @@ type SubmitDistributedQueryResultsFunc func(ctx context.Context, results kolide. type SubmitStatusLogsFunc func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) -type SubmitResultLogsFunc func(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) +type SubmitResultLogsFunc func(ctx context.Context, logs []json.RawMessage) (err error) type TLSService struct { EnrollAgentFunc EnrollAgentFunc @@ -77,7 +78,7 @@ func (s *TLSService) SubmitStatusLogs(ctx context.Context, logs []kolide.Osquery return s.SubmitStatusLogsFunc(ctx, logs) } -func (s *TLSService) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) { +func (s *TLSService) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error) { s.SubmitResultLogsFuncInvoked = true return s.SubmitResultLogsFunc(ctx, logs) } diff --git a/server/service/endpoint_osquery.go b/server/service/endpoint_osquery.go index 95e7e9b78..5519e258f 100644 --- a/server/service/endpoint_osquery.go +++ b/server/service/endpoint_osquery.go @@ -148,7 +148,7 @@ func makeSubmitLogsEndpoint(svc kolide.Service) endpoint.Endpoint { } case "result": - var results []kolide.OsqueryResultLog + var results []json.RawMessage if err := json.Unmarshal(req.Data, &results); err != nil { err = osqueryError{message: "unmarshalling result logs: " + err.Error()} break diff --git a/server/service/logging_osquery.go b/server/service/logging_osquery.go index 6e5449cd0..ca4e642ff 100644 --- a/server/service/logging_osquery.go +++ b/server/service/logging_osquery.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "time" kithttp "github.com/go-kit/kit/transport/http" @@ -121,7 +122,7 @@ func (mw loggingMiddleware) SubmitStatusLogs(ctx context.Context, logs []kolide. return err } -func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error { +func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error { var ( err error ) diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index e207e4495..dfdce1ee8 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -236,10 +236,9 @@ func (svc service) SubmitStatusLogs(ctx context.Context, logs []kolide.OsquerySt return nil } -func (svc service) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error { +func (svc service) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error { for _, log := range logs { - err := json.NewEncoder(svc.osqueryResultLogWriter).Encode(log) - if err != nil { + if _, err := svc.osqueryResultLogWriter.Write(append(log, '\n')); err != nil { return osqueryError{message: "error writing result log: " + err.Error()} } } diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index a44457c37..0f188477c 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -159,10 +159,13 @@ func TestSubmitResultLogs(t *testing.T) { `{"name":"system_info","hostIdentifier":"some_uuid","calendarTime":"Fri Sep 30 17:55:15 2016 UTC","unixTime":"1475258115","decorations":{"host_uuid":"some_uuid","username":"zwass"},"columns":{"cpu_brand":"Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz","hostname":"hostimus","physical_memory":"17179869184"},"action":"added"}`, `{"name":"encrypted","hostIdentifier":"some_uuid","calendarTime":"Fri Sep 30 21:19:15 2016 UTC","unixTime":"1475270355","decorations":{"host_uuid":"4740D59F-699E-5B29-960B-979AAF9BBEEB","username":"zwass"},"columns":{"encrypted":"1","name":"\/dev\/disk1","type":"AES-XTS","uid":"","user_uuid":"","uuid":"some_uuid"},"action":"added"}`, `{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"time","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:51 2017 UTC","unixTime":"1484078931","decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`, + `{"diffResults":{"removed":[{"address":"127.0.0.1","hostnames":"kl.groob.io"}],"added":""},"name":"pack\/test\/hosts","hostIdentifier":"FA01680E-98CA-5557-8F59-7716ECFEE964","calendarTime":"Sun Nov 19 00:02:08 2017 UTC","unixTime":"1511049728","epoch":"0","counter":"10","decorations":{"host_uuid":"FA01680E-98CA-5557-8F59-7716ECFEE964","hostname":"kl.groob.io"}}`, + // fleet will accept anything in the "data" field of an log request. + `{"unknown":{"foo": [] }}`, } logJSON := fmt.Sprintf("[%s]", strings.Join(logs, ",")) - var results []kolide.OsqueryResultLog + var results []json.RawMessage err = json.Unmarshal([]byte(logJSON), &results) require.Nil(t, err)