mirror of
https://github.com/empayre/fleet.git
synced 2024-11-06 17:05:18 +00:00
6415f062c6
This was found while working on #10957. When running a live query, a lot of unused host data is stored in Redis and sent on every live query result message via websockets. The frontend and fleetctl just need `id`, `hostname` and `display_name`. (This becomes worse every time we add new fields to the `Host` struct.) Sample of one websocket message result when running `SELECT * from osquery_info;`: size in `main`: 2234 bytes ``` a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":57,\"host\": {\"created_at\":\"2023-05-22T12:14:11Z\",\"updated_at\":\"2023-05-23T12:31:51Z\", \"software_updated_at\":\"0001-01-01T00:00:00Z\",\"id\":106,\"detail_updated_at\":\"2023-05-23T11:50:04Z\", \"label_updated_at\":\"2023-05-23T11:50:04Z\",\"policy_updated_at\":\"1970-01-02T00:00:00Z\", \"last_enrolled_at\":\"2023-05-22T12:14:12Z\", \"seen_time\":\"2023-05-23T09:52:23.876311-03:00\",\"refetch_requested\":false, \"hostname\":\"lucass-macbook-pro.local\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"platform\":\"darwin\",\"osquery_version\":\"5.8.2\",\"os_version\":\"macOS 13.3.1\",\"build\":\"22E261\",\"platform_like\":\"darwin\",\"code_name\":\"\", \"uptime\":91125000000000,\"memory\":34359738368,\"cpu_type\":\"x86_64h\",\"cpu_subtype\":\"Intel x86-64h Haswell\",\"cpu_brand\":\"Intel(R) Core(TM) i7-1068NG7 CPU @ 2.30GHz\",\"cpu_physical_cores\":4,\"cpu_logical_cores\":8,\"hardware_vendor\":\"Apple Inc.\",\"hardware_model\":\"MacBookPro16,2\",\"hardware_version\":\"1.0\", \"hardware_serial\":\"0DPQR4HMD1FZ\", \"computer_name\":\"Lucas’s MacBook Pro\",\"public_ip\":\"\", \"primary_ip\":\"192.168.0.230\",\"primary_mac\":\"68:2f:67:8e:b6:1f\", \"distributed_interval\":1,\"config_tls_refresh\":60,\"logger_tls_period\":10,\"team_id\":null, \"pack_stats\":null,\"team_name\":null, \"gigs_disk_space_available\":386.23,\"percent_disk_space_available\":40, \"issues\":{\"total_issues_count\":0,\"failing_policies_count\":0}, \"mdm\":{\"enrollment_status\":null,\"server_url\":null,\"name\":\"\",\"encryption_key_available\":false}, \"status\":\"online\",\"display_text\":\"lucass-macbook-pro.local\",\"display_name\":\"Lucas’s MacBook Pro\"}, \"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\", \"config_hash\":\"b7ee9363a7c686e76e99ffb122e9c5241a791e69\",\"config_valid\":\"1\", \"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\", \"host_hostname\":\"lucass-macbook-pro.local\",\"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\",\"start_time\":\"1684757652\", \"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\", \"version\":\"5.8.2\",\"watcher\":\"8364\"}],\"error\":null}}"] ``` vs. size of the message result on this branch: 675 bytes ``` a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":59, \"host\":{\"id\":106,\"hostname\":\"lucass-macbook-pro.local\", \"display_name\":\"Lucas’s MacBook Pro\"}, \"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\", \"config_hash\":\"f80dee827635db39077a458243379b3ad63311fd\", \"config_valid\":\"1\",\"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\", \"host_hostname\":\"lucass-macbook-pro.local\", \"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\", \"start_time\":\"1684757652\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"version\":\"5.8.2\", \"watcher\":\"8364\"}]}}"] ``` Manual tests included running with an old fleetctl running with a new fleet server, and vice-versa, a new fleetctl running against an old fleet server. - [X] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [X] Documented any API changes (docs/Using-Fleet/REST-API.md or docs/Contributing/API-for-contributors.md) - ~[ ] Documented any permissions changes~ - ~[ ] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements)~ - ~[ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features.~ - [X] Added/updated tests - [X] Manual QA for all new/changed functionality - ~For Orbit and Fleet Desktop changes:~ - ~[ ] Manual QA must be performed in the three main OSs, macOS, Windows and Linux.~ - ~[ ] Auto-update manual QA, from released version of component to new version (see [tools/tuf/test](../tools/tuf/test/README.md)).~
115 lines
3.7 KiB
Go
115 lines
3.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/fleetdm/fleet/v4/server/fleet"
|
|
"github.com/fleetdm/fleet/v4/server/live_query/live_query_mock"
|
|
"github.com/fleetdm/fleet/v4/server/pubsub"
|
|
"github.com/fleetdm/fleet/v4/server/service"
|
|
kitlog "github.com/go-kit/kit/log"
|
|
"github.com/go-kit/kit/log/level"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestLiveQuery(t *testing.T) {
|
|
rs := pubsub.NewInmemQueryResults()
|
|
lq := live_query_mock.New(t)
|
|
|
|
logger := kitlog.NewJSONLogger(os.Stdout)
|
|
logger = level.NewFilter(logger, level.AllowDebug())
|
|
|
|
_, ds := runServerWithMockedDS(t, &service.TestServerOpts{
|
|
Rs: rs,
|
|
Lq: lq,
|
|
Logger: logger,
|
|
})
|
|
|
|
users, err := ds.ListUsersFunc(context.Background(), fleet.UserListOptions{})
|
|
require.NoError(t, err)
|
|
var admin *fleet.User
|
|
for _, user := range users {
|
|
if user.GlobalRole != nil && *user.GlobalRole == fleet.RoleAdmin {
|
|
admin = user
|
|
}
|
|
}
|
|
|
|
ds.HostIDsByNameFunc = func(ctx context.Context, filter fleet.TeamFilter, hostnames []string) ([]uint, error) {
|
|
return []uint{1234}, nil
|
|
}
|
|
ds.LabelIDsByNameFunc = func(ctx context.Context, labels []string) ([]uint, error) {
|
|
return nil, nil
|
|
}
|
|
ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
|
|
return &fleet.AppConfig{}, nil
|
|
}
|
|
ds.NewQueryFunc = func(ctx context.Context, query *fleet.Query, opts ...fleet.OptionalArg) (*fleet.Query, error) {
|
|
query.ID = 42
|
|
return query, nil
|
|
}
|
|
ds.NewDistributedQueryCampaignFunc = func(ctx context.Context, camp *fleet.DistributedQueryCampaign) (*fleet.DistributedQueryCampaign, error) {
|
|
camp.ID = 321
|
|
return camp, nil
|
|
}
|
|
ds.NewDistributedQueryCampaignTargetFunc = func(ctx context.Context, target *fleet.DistributedQueryCampaignTarget) (*fleet.DistributedQueryCampaignTarget, error) {
|
|
return target, nil
|
|
}
|
|
ds.HostIDsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets) ([]uint, error) {
|
|
return []uint{1}, nil
|
|
}
|
|
ds.CountHostsInTargetsFunc = func(ctx context.Context, filter fleet.TeamFilter, targets fleet.HostTargets, now time.Time) (fleet.TargetMetrics, error) {
|
|
return fleet.TargetMetrics{TotalHosts: 1, OnlineHosts: 1}, nil
|
|
}
|
|
ds.NewActivityFunc = func(ctx context.Context, user *fleet.User, activity fleet.ActivityDetails) error {
|
|
return nil
|
|
}
|
|
|
|
lq.On("QueriesForHost", uint(1)).Return(
|
|
map[string]string{
|
|
"42": "select 42, * from time",
|
|
},
|
|
nil,
|
|
)
|
|
lq.On("QueryCompletedByHost", "42", 99).Return(nil)
|
|
lq.On("RunQuery", "321", "select 42, * from time", []uint{1}).Return(nil)
|
|
|
|
ds.DistributedQueryCampaignTargetIDsFunc = func(ctx context.Context, id uint) (targets *fleet.HostTargets, err error) {
|
|
return &fleet.HostTargets{HostIDs: []uint{99}}, nil
|
|
}
|
|
ds.DistributedQueryCampaignFunc = func(ctx context.Context, id uint) (*fleet.DistributedQueryCampaign, error) {
|
|
return &fleet.DistributedQueryCampaign{
|
|
ID: 321,
|
|
UserID: admin.ID,
|
|
}, nil
|
|
}
|
|
ds.SaveDistributedQueryCampaignFunc = func(ctx context.Context, camp *fleet.DistributedQueryCampaign) error {
|
|
return nil
|
|
}
|
|
ds.QueryFunc = func(ctx context.Context, id uint) (*fleet.Query, error) {
|
|
return &fleet.Query{}, nil
|
|
}
|
|
|
|
go func() {
|
|
time.Sleep(2 * time.Second)
|
|
require.NoError(t, rs.WriteResult(
|
|
fleet.DistributedQueryResult{
|
|
DistributedQueryCampaignID: 321,
|
|
Rows: []map[string]string{{"bing": "fds"}},
|
|
Host: fleet.ResultHostData{
|
|
ID: 99,
|
|
Hostname: "somehostname",
|
|
DisplayName: "somehostname",
|
|
},
|
|
},
|
|
))
|
|
}()
|
|
|
|
expected := `{"host":"somehostname","rows":[{"bing":"fds","host_display_name":"somehostname","host_hostname":"somehostname"}]}
|
|
`
|
|
assert.Equal(t, expected, runAppForTest(t, []string{"query", "--hosts", "1234", "--query", "select 42, * from time"}))
|
|
}
|