fleet/server/service/client_live_query_test.go

122 lines
2.9 KiB
Go
Raw Normal View History

package service
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"time"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/gorilla/websocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLiveQueryWithContext(t *testing.T) {
upgrader := websocket.Upgrader{}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/latest/fleet/queries/run_by_names":
resp := createDistributedQueryCampaignResponse{
Campaign: &fleet.DistributedQueryCampaign{
UpdateCreateTimestamps: fleet.UpdateCreateTimestamps{
CreateTimestamp: fleet.CreateTimestamp{CreatedAt: time.Now()},
UpdateTimestamp: fleet.UpdateTimestamp{UpdatedAt: time.Now()},
},
Metrics: fleet.TargetMetrics{
TotalHosts: 1,
OnlineHosts: 1,
OfflineHosts: 0,
MissingInActionHosts: 0,
NewHosts: 0,
},
ID: 99,
QueryID: 42,
Status: 0,
UserID: 23,
},
}
err := json.NewEncoder(w).Encode(resp)
assert.NoError(t, err)
case "/api/latest/fleet/results/websocket":
ws, _ := upgrader.Upgrade(w, r, nil)
defer ws.Close()
for {
time.Sleep(1 * time.Second)
mt, message, _ := ws.ReadMessage()
if string(message) == `{"type":"auth","data":{"token":"1234"}}` {
return
}
if string(message) == `{"type":"select_campaign","data":{"campaign_id":99}}` {
return
}
result := struct {
Type string `json:"type"`
Data fleet.DistributedQueryResult `json:"data"`
}{
Type: "result",
Data: fleet.DistributedQueryResult{
DistributedQueryCampaignID: 99,
Reduce size of `DistributedQueryResult` to improve live query performance (#11882) This was found while working on #10957. When running a live query, a lot of unused host data is stored in Redis and sent on every live query result message via websockets. The frontend and fleetctl just need `id`, `hostname` and `display_name`. (This becomes worse every time we add new fields to the `Host` struct.) Sample of one websocket message result when running `SELECT * from osquery_info;`: size in `main`: 2234 bytes ``` a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":57,\"host\": {\"created_at\":\"2023-05-22T12:14:11Z\",\"updated_at\":\"2023-05-23T12:31:51Z\", \"software_updated_at\":\"0001-01-01T00:00:00Z\",\"id\":106,\"detail_updated_at\":\"2023-05-23T11:50:04Z\", \"label_updated_at\":\"2023-05-23T11:50:04Z\",\"policy_updated_at\":\"1970-01-02T00:00:00Z\", \"last_enrolled_at\":\"2023-05-22T12:14:12Z\", \"seen_time\":\"2023-05-23T09:52:23.876311-03:00\",\"refetch_requested\":false, \"hostname\":\"lucass-macbook-pro.local\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"platform\":\"darwin\",\"osquery_version\":\"5.8.2\",\"os_version\":\"macOS 13.3.1\",\"build\":\"22E261\",\"platform_like\":\"darwin\",\"code_name\":\"\", \"uptime\":91125000000000,\"memory\":34359738368,\"cpu_type\":\"x86_64h\",\"cpu_subtype\":\"Intel x86-64h Haswell\",\"cpu_brand\":\"Intel(R) Core(TM) i7-1068NG7 CPU @ 2.30GHz\",\"cpu_physical_cores\":4,\"cpu_logical_cores\":8,\"hardware_vendor\":\"Apple Inc.\",\"hardware_model\":\"MacBookPro16,2\",\"hardware_version\":\"1.0\", \"hardware_serial\":\"0DPQR4HMD1FZ\", \"computer_name\":\"Lucas’s MacBook Pro\",\"public_ip\":\"\", \"primary_ip\":\"192.168.0.230\",\"primary_mac\":\"68:2f:67:8e:b6:1f\", \"distributed_interval\":1,\"config_tls_refresh\":60,\"logger_tls_period\":10,\"team_id\":null, \"pack_stats\":null,\"team_name\":null, \"gigs_disk_space_available\":386.23,\"percent_disk_space_available\":40, \"issues\":{\"total_issues_count\":0,\"failing_policies_count\":0}, \"mdm\":{\"enrollment_status\":null,\"server_url\":null,\"name\":\"\",\"encryption_key_available\":false}, \"status\":\"online\",\"display_text\":\"lucass-macbook-pro.local\",\"display_name\":\"Lucas’s MacBook Pro\"}, \"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\", \"config_hash\":\"b7ee9363a7c686e76e99ffb122e9c5241a791e69\",\"config_valid\":\"1\", \"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\", \"host_hostname\":\"lucass-macbook-pro.local\",\"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\",\"start_time\":\"1684757652\", \"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\", \"version\":\"5.8.2\",\"watcher\":\"8364\"}],\"error\":null}}"] ``` vs. size of the message result on this branch: 675 bytes ``` a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":59, \"host\":{\"id\":106,\"hostname\":\"lucass-macbook-pro.local\", \"display_name\":\"Lucas’s MacBook Pro\"}, \"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\", \"config_hash\":\"f80dee827635db39077a458243379b3ad63311fd\", \"config_valid\":\"1\",\"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\", \"host_hostname\":\"lucass-macbook-pro.local\", \"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\", \"start_time\":\"1684757652\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"version\":\"5.8.2\", \"watcher\":\"8364\"}]}}"] ``` Manual tests included running with an old fleetctl running with a new fleet server, and vice-versa, a new fleetctl running against an old fleet server. - [X] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [X] Documented any API changes (docs/Using-Fleet/REST-API.md or docs/Contributing/API-for-contributors.md) - ~[ ] Documented any permissions changes~ - ~[ ] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements)~ - ~[ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features.~ - [X] Added/updated tests - [X] Manual QA for all new/changed functionality - ~For Orbit and Fleet Desktop changes:~ - ~[ ] Manual QA must be performed in the three main OSs, macOS, Windows and Linux.~ - ~[ ] Auto-update manual QA, from released version of component to new version (see [tools/tuf/test](../tools/tuf/test/README.md)).~
2023-05-25 11:11:53 +00:00
Host: fleet.ResultHostData{
ID: 23,
Hostname: "somehostaaa",
Reduce size of `DistributedQueryResult` to improve live query performance (#11882) This was found while working on #10957. When running a live query, a lot of unused host data is stored in Redis and sent on every live query result message via websockets. The frontend and fleetctl just need `id`, `hostname` and `display_name`. (This becomes worse every time we add new fields to the `Host` struct.) Sample of one websocket message result when running `SELECT * from osquery_info;`: size in `main`: 2234 bytes ``` a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":57,\"host\": {\"created_at\":\"2023-05-22T12:14:11Z\",\"updated_at\":\"2023-05-23T12:31:51Z\", \"software_updated_at\":\"0001-01-01T00:00:00Z\",\"id\":106,\"detail_updated_at\":\"2023-05-23T11:50:04Z\", \"label_updated_at\":\"2023-05-23T11:50:04Z\",\"policy_updated_at\":\"1970-01-02T00:00:00Z\", \"last_enrolled_at\":\"2023-05-22T12:14:12Z\", \"seen_time\":\"2023-05-23T09:52:23.876311-03:00\",\"refetch_requested\":false, \"hostname\":\"lucass-macbook-pro.local\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"platform\":\"darwin\",\"osquery_version\":\"5.8.2\",\"os_version\":\"macOS 13.3.1\",\"build\":\"22E261\",\"platform_like\":\"darwin\",\"code_name\":\"\", \"uptime\":91125000000000,\"memory\":34359738368,\"cpu_type\":\"x86_64h\",\"cpu_subtype\":\"Intel x86-64h Haswell\",\"cpu_brand\":\"Intel(R) Core(TM) i7-1068NG7 CPU @ 2.30GHz\",\"cpu_physical_cores\":4,\"cpu_logical_cores\":8,\"hardware_vendor\":\"Apple Inc.\",\"hardware_model\":\"MacBookPro16,2\",\"hardware_version\":\"1.0\", \"hardware_serial\":\"0DPQR4HMD1FZ\", \"computer_name\":\"Lucas’s MacBook Pro\",\"public_ip\":\"\", \"primary_ip\":\"192.168.0.230\",\"primary_mac\":\"68:2f:67:8e:b6:1f\", \"distributed_interval\":1,\"config_tls_refresh\":60,\"logger_tls_period\":10,\"team_id\":null, \"pack_stats\":null,\"team_name\":null, \"gigs_disk_space_available\":386.23,\"percent_disk_space_available\":40, \"issues\":{\"total_issues_count\":0,\"failing_policies_count\":0}, \"mdm\":{\"enrollment_status\":null,\"server_url\":null,\"name\":\"\",\"encryption_key_available\":false}, \"status\":\"online\",\"display_text\":\"lucass-macbook-pro.local\",\"display_name\":\"Lucas’s MacBook Pro\"}, \"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\", \"config_hash\":\"b7ee9363a7c686e76e99ffb122e9c5241a791e69\",\"config_valid\":\"1\", \"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\", \"host_hostname\":\"lucass-macbook-pro.local\",\"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\",\"start_time\":\"1684757652\", \"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\", \"version\":\"5.8.2\",\"watcher\":\"8364\"}],\"error\":null}}"] ``` vs. size of the message result on this branch: 675 bytes ``` a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":59, \"host\":{\"id\":106,\"hostname\":\"lucass-macbook-pro.local\", \"display_name\":\"Lucas’s MacBook Pro\"}, \"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\", \"config_hash\":\"f80dee827635db39077a458243379b3ad63311fd\", \"config_valid\":\"1\",\"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\", \"host_hostname\":\"lucass-macbook-pro.local\", \"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\", \"start_time\":\"1684757652\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"version\":\"5.8.2\", \"watcher\":\"8364\"}]}}"] ``` Manual tests included running with an old fleetctl running with a new fleet server, and vice-versa, a new fleetctl running against an old fleet server. - [X] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [X] Documented any API changes (docs/Using-Fleet/REST-API.md or docs/Contributing/API-for-contributors.md) - ~[ ] Documented any permissions changes~ - ~[ ] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements)~ - ~[ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features.~ - [X] Added/updated tests - [X] Manual QA for all new/changed functionality - ~For Orbit and Fleet Desktop changes:~ - ~[ ] Manual QA must be performed in the three main OSs, macOS, Windows and Linux.~ - ~[ ] Auto-update manual QA, from released version of component to new version (see [tools/tuf/test](../tools/tuf/test/README.md)).~
2023-05-25 11:11:53 +00:00
},
Rows: []map[string]string{
{
"col1": "aaa",
"col2": "bbb",
},
},
Error: nil,
},
}
b, err := json.Marshal(result)
assert.NoError(t, err)
_ = ws.WriteMessage(mt, b)
}
}
}))
defer ts.Close()
baseURL, err := url.Parse(ts.URL)
require.NoError(t, err)
client := &Client{
baseClient: &baseClient{
baseURL: baseURL,
http: fleethttp.NewClient(),
insecureSkipVerify: false,
urlPrefix: "",
},
token: "1234",
outputWriter: nil,
}
ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Second)
defer cancelFunc()
res, err := client.LiveQueryWithContext(ctx, "select 1;", nil, nil, []string{"host1"})
require.NoError(t, err)
gotResults := false
go func() {
for {
select {
case <-res.Results():
gotResults = true
cancelFunc()
case err := <-res.Errors():
require.NoError(t, err)
case <-ctx.Done():
return
}
}
}()
<-ctx.Done()
assert.True(t, gotResults)
}