empayre/fleet
14
0
Fork 0
mirror of https://github.com/empayre/fleet.git synced 2024-11-06 08:55:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

use json.RawMessage for result logs (#1636)

Browse Source 
Initially fleet decoded the incoming JSON sent to the log endpoint.
Then the log event would be written to a log writer by calling json.Encoder{}.Encode.
Re-encoding logs is lossy; whenever a new field is sent by osqueryd we don't keep up with them.
Instead of caring about the content of the OsqueryResultLog, fleet will now write all log results
exactly as sent to the server by osqueryd.

Closes #1632
Closes #1615
This commit is contained in:
Victor Vrantchan 2017-11-18 19:59:32 -05:00 committed by GitHub
parent b2771b80c0
commit 8291119067
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 22 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,3 +1,7 @@
## Kolide Fleet 1.0.6 (TBD, 2017)
* Fixed bugs that caused logs to sometimes be ommited from the logwriter. (#1636, #1617)
* Fixed a bug where SAML client would create too many HTTP connections. (#1587)
* Fixed bug in which default query was run instead of entered query. (#1611)

18
server/kolide/osquery.go
View File

@ -2,6 +2,7 @@ package kolide
import (
"context"
"encoding/json"
)
type OsqueryService interface {
@ -18,7 +19,7 @@ type OsqueryService interface {
GetDistributedQueries(ctx context.Context) (queries map[string]string, accelerate uint, err error)
SubmitDistributedQueryResults(ctx context.Context, results OsqueryDistributedQueryResults, statuses map[string]string) (err error)
SubmitStatusLogs(ctx context.Context, logs []OsqueryStatusLog) (err error)
SubmitResultLogs(ctx context.Context, logs []OsqueryResultLog) (err error)
SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error)
}
// OsqueryDistributedQueryResults represents the format of the results of an
@ -71,21 +72,6 @@ type OsqueryConfig struct {
FilePaths FIMSections `json:"file_paths,omitempty"`
}
// OsqueryResultLog is the format of an osquery result log (ie: a differential
// or snapshot query).
type OsqueryResultLog struct {
Name string `json:"name"`
HostIdentifier string `json:"hostIdentifier"`
UnixTime string `json:"unixTime"`
CalendarTime string `json:"calendarTime"`
// Columns stores the columns of differential queries
Columns map[string]string `json:"columns,omitempty"`
// Snapshot stores the rows and columns of snapshot queries
Snapshot []map[string]string `json:"snapshot,omitempty"`
Action string `json:"action"`
Decorations map[string]string `json:"decorations"`
}
// OsqueryStatusLog is the format of an osquery status log.
type OsqueryStatusLog struct {
Severity string `json:"severity"`

8
server/launcher/launcher.go
View File

@ -112,13 +112,9 @@ func (svc *launcherWrapper) PublishLogs(ctx context.Context, nodeKey string, log
err = svc.tls.SubmitStatusLogs(newCtx, statuses)
return "", "", false, errors.Wrap(err, "submit status logs from launcher")
case logger.LogTypeSnapshot, logger.LogTypeString:
var results []kolide.OsqueryResultLog
var results []json.RawMessage
for _, log := range logs {
var result kolide.OsqueryResultLog
if err := json.Unmarshal([]byte(log), &result); err != nil {
return "", "", false, errors.Wrap(err, "unmarshaling result log")
}
results = append(results, result)
results = append(results, []byte(log))
}
err = svc.tls.SubmitResultLogs(newCtx, results)
return "", "", false, errors.Wrap(err, "submit result logs from launcher")

3
server/launcher/launcher_test.go
View File

@ -2,6 +2,7 @@ package launcher
import (
"context"
"encoding/json"
"testing"
"github.com/go-kit/kit/log"
@ -148,7 +149,7 @@ func newTLSService(t *testing.T) *mock.TLSService {
SubmitStatusLogsFunc: func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error) {
return
},
SubmitResultLogsFunc: func(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) {
SubmitResultLogsFunc: func(ctx context.Context, logs []json.RawMessage) (err error) {
return
},
}

5
server/mock/service_osquery.go
View File

@ -4,6 +4,7 @@ package mock
import (
"context"
"encoding/json"
"github.com/kolide/fleet/server/kolide"
)
@ -22,7 +23,7 @@ type SubmitDistributedQueryResultsFunc func(ctx context.Context, results kolide.
type SubmitStatusLogsFunc func(ctx context.Context, logs []kolide.OsqueryStatusLog) (err error)
type SubmitResultLogsFunc func(ctx context.Context, logs []kolide.OsqueryResultLog) (err error)
type SubmitResultLogsFunc func(ctx context.Context, logs []json.RawMessage) (err error)
type TLSService struct {
EnrollAgentFunc EnrollAgentFunc
@ -77,7 +78,7 @@ func (s *TLSService) SubmitStatusLogs(ctx context.Context, logs []kolide.Osquery
return s.SubmitStatusLogsFunc(ctx, logs)
}
func (s *TLSService) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) (err error) {
func (s *TLSService) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error) {
s.SubmitResultLogsFuncInvoked = true
return s.SubmitResultLogsFunc(ctx, logs)
}

2
server/service/endpoint_osquery.go
View File

@ -148,7 +148,7 @@ func makeSubmitLogsEndpoint(svc kolide.Service) endpoint.Endpoint {
}
case "result":
var results []kolide.OsqueryResultLog
var results []json.RawMessage
if err := json.Unmarshal(req.Data, &results); err != nil {
err = osqueryError{message: "unmarshalling result logs: " + err.Error()}
break

3
server/service/logging_osquery.go
View File

@ -2,6 +2,7 @@ package service
import (
"context"
"encoding/json"
"time"
kithttp "github.com/go-kit/kit/transport/http"
@ -121,7 +122,7 @@ func (mw loggingMiddleware) SubmitStatusLogs(ctx context.Context, logs []kolide.
return err
}
func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error {
func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error {
var (
err error
)

5
server/service/service_osquery.go
View File

@ -236,10 +236,9 @@ func (svc service) SubmitStatusLogs(ctx context.Context, logs []kolide.OsquerySt
return nil
}
func (svc service) SubmitResultLogs(ctx context.Context, logs []kolide.OsqueryResultLog) error {
func (svc service) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error {
for _, log := range logs {
err := json.NewEncoder(svc.osqueryResultLogWriter).Encode(log)
if err != nil {
if _, err := svc.osqueryResultLogWriter.Write(append(log, '\n')); err != nil {
return osqueryError{message: "error writing result log: " + err.Error()}
}
}

5
server/service/service_osquery_test.go
View File

@ -159,10 +159,13 @@ func TestSubmitResultLogs(t *testing.T) {
`{"name":"system_info","hostIdentifier":"some_uuid","calendarTime":"Fri Sep 30 17:55:15 2016 UTC","unixTime":"1475258115","decorations":{"host_uuid":"some_uuid","username":"zwass"},"columns":{"cpu_brand":"Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz","hostname":"hostimus","physical_memory":"17179869184"},"action":"added"}`,
`{"name":"encrypted","hostIdentifier":"some_uuid","calendarTime":"Fri Sep 30 21:19:15 2016 UTC","unixTime":"1475270355","decorations":{"host_uuid":"4740D59F-699E-5B29-960B-979AAF9BBEEB","username":"zwass"},"columns":{"encrypted":"1","name":"\/dev\/disk1","type":"AES-XTS","uid":"","user_uuid":"","uuid":"some_uuid"},"action":"added"}`,
`{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"time","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:51 2017 UTC","unixTime":"1484078931","decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`,
`{"diffResults":{"removed":[{"address":"127.0.0.1","hostnames":"kl.groob.io"}],"added":""},"name":"pack\/test\/hosts","hostIdentifier":"FA01680E-98CA-5557-8F59-7716ECFEE964","calendarTime":"Sun Nov 19 00:02:08 2017 UTC","unixTime":"1511049728","epoch":"0","counter":"10","decorations":{"host_uuid":"FA01680E-98CA-5557-8F59-7716ECFEE964","hostname":"kl.groob.io"}}`,
// fleet will accept anything in the "data" field of an log request.
`{"unknown":{"foo": [] }}`,
}
logJSON := fmt.Sprintf("[%s]", strings.Join(logs, ","))
var results []kolide.OsqueryResultLog
var results []json.RawMessage
err = json.Unmarshal([]byte(logJSON), &results)
require.Nil(t, err)