2021-09-10 19:26:39 +00:00
|
|
|
package service
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"net/http"
|
|
|
|
"net/http/httptest"
|
|
|
|
"net/url"
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
2021-11-24 20:56:54 +00:00
|
|
|
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
|
2021-09-10 19:26:39 +00:00
|
|
|
"github.com/fleetdm/fleet/v4/server/fleet"
|
|
|
|
"github.com/gorilla/websocket"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestLiveQueryWithContext(t *testing.T) {
|
|
|
|
upgrader := websocket.Upgrader{}
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
|
|
switch r.URL.Path {
|
2022-04-05 15:35:53 +00:00
|
|
|
case "/api/latest/fleet/queries/run_by_names":
|
2021-09-10 19:26:39 +00:00
|
|
|
resp := createDistributedQueryCampaignResponse{
|
|
|
|
Campaign: &fleet.DistributedQueryCampaign{
|
|
|
|
UpdateCreateTimestamps: fleet.UpdateCreateTimestamps{
|
|
|
|
CreateTimestamp: fleet.CreateTimestamp{CreatedAt: time.Now()},
|
|
|
|
UpdateTimestamp: fleet.UpdateTimestamp{UpdatedAt: time.Now()},
|
|
|
|
},
|
|
|
|
Metrics: fleet.TargetMetrics{
|
|
|
|
TotalHosts: 1,
|
|
|
|
OnlineHosts: 1,
|
|
|
|
OfflineHosts: 0,
|
|
|
|
MissingInActionHosts: 0,
|
|
|
|
NewHosts: 0,
|
|
|
|
},
|
|
|
|
ID: 99,
|
|
|
|
QueryID: 42,
|
|
|
|
Status: 0,
|
|
|
|
UserID: 23,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
err := json.NewEncoder(w).Encode(resp)
|
|
|
|
assert.NoError(t, err)
|
2022-04-20 19:57:26 +00:00
|
|
|
case "/api/latest/fleet/results/websocket":
|
2021-09-10 19:26:39 +00:00
|
|
|
ws, _ := upgrader.Upgrade(w, r, nil)
|
|
|
|
defer ws.Close()
|
|
|
|
|
|
|
|
for {
|
|
|
|
time.Sleep(1 * time.Second)
|
|
|
|
mt, message, _ := ws.ReadMessage()
|
|
|
|
if string(message) == `{"type":"auth","data":{"token":"1234"}}` {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if string(message) == `{"type":"select_campaign","data":{"campaign_id":99}}` {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
result := struct {
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data fleet.DistributedQueryResult `json:"data"`
|
|
|
|
}{
|
|
|
|
Type: "result",
|
|
|
|
Data: fleet.DistributedQueryResult{
|
|
|
|
DistributedQueryCampaignID: 99,
|
Reduce size of `DistributedQueryResult` to improve live query performance (#11882)
This was found while working on #10957.
When running a live query, a lot of unused host data is stored in Redis
and sent on every live query result message via websockets. The frontend
and fleetctl just need `id`, `hostname` and `display_name`. (This
becomes worse every time we add new fields to the `Host` struct.)
Sample of one websocket message result when running `SELECT * from
osquery_info;`:
size in `main`: 2234 bytes
```
a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":57,\"host\":
{\"created_at\":\"2023-05-22T12:14:11Z\",\"updated_at\":\"2023-05-23T12:31:51Z\",
\"software_updated_at\":\"0001-01-01T00:00:00Z\",\"id\":106,\"detail_updated_at\":\"2023-05-23T11:50:04Z\",
\"label_updated_at\":\"2023-05-23T11:50:04Z\",\"policy_updated_at\":\"1970-01-02T00:00:00Z\",
\"last_enrolled_at\":\"2023-05-22T12:14:12Z\",
\"seen_time\":\"2023-05-23T09:52:23.876311-03:00\",\"refetch_requested\":false,
\"hostname\":\"lucass-macbook-pro.local\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"platform\":\"darwin\",\"osquery_version\":\"5.8.2\",\"os_version\":\"macOS 13.3.1\",\"build\":\"22E261\",\"platform_like\":\"darwin\",\"code_name\":\"\",
\"uptime\":91125000000000,\"memory\":34359738368,\"cpu_type\":\"x86_64h\",\"cpu_subtype\":\"Intel x86-64h Haswell\",\"cpu_brand\":\"Intel(R) Core(TM) i7-1068NG7 CPU @ 2.30GHz\",\"cpu_physical_cores\":4,\"cpu_logical_cores\":8,\"hardware_vendor\":\"Apple Inc.\",\"hardware_model\":\"MacBookPro16,2\",\"hardware_version\":\"1.0\",
\"hardware_serial\":\"0DPQR4HMD1FZ\",
\"computer_name\":\"Lucas’s MacBook Pro\",\"public_ip\":\"\",
\"primary_ip\":\"192.168.0.230\",\"primary_mac\":\"68:2f:67:8e:b6:1f\",
\"distributed_interval\":1,\"config_tls_refresh\":60,\"logger_tls_period\":10,\"team_id\":null,
\"pack_stats\":null,\"team_name\":null,
\"gigs_disk_space_available\":386.23,\"percent_disk_space_available\":40,
\"issues\":{\"total_issues_count\":0,\"failing_policies_count\":0},
\"mdm\":{\"enrollment_status\":null,\"server_url\":null,\"name\":\"\",\"encryption_key_available\":false},
\"status\":\"online\",\"display_text\":\"lucass-macbook-pro.local\",\"display_name\":\"Lucas’s MacBook Pro\"},
\"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\",
\"config_hash\":\"b7ee9363a7c686e76e99ffb122e9c5241a791e69\",\"config_valid\":\"1\",
\"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\",
\"host_hostname\":\"lucass-macbook-pro.local\",\"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\",\"start_time\":\"1684757652\",
\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",
\"version\":\"5.8.2\",\"watcher\":\"8364\"}],\"error\":null}}"]
```
vs. size of the message result on this branch: 675 bytes
```
a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":59,
\"host\":{\"id\":106,\"hostname\":\"lucass-macbook-pro.local\",
\"display_name\":\"Lucas’s MacBook Pro\"},
\"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\",
\"config_hash\":\"f80dee827635db39077a458243379b3ad63311fd\",
\"config_valid\":\"1\",\"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\",
\"host_hostname\":\"lucass-macbook-pro.local\",
\"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\",
\"start_time\":\"1684757652\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"version\":\"5.8.2\",
\"watcher\":\"8364\"}]}}"]
```
Manual tests included running with an old fleetctl running with a new
fleet server, and vice-versa, a new fleetctl running against an old
fleet server.
- [X] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [X] Documented any API changes (docs/Using-Fleet/REST-API.md or
docs/Contributing/API-for-contributors.md)
- ~[ ] Documented any permissions changes~
- ~[ ] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)~
- ~[ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for
new osquery data ingestion features.~
- [X] Added/updated tests
- [X] Manual QA for all new/changed functionality
- ~For Orbit and Fleet Desktop changes:~
- ~[ ] Manual QA must be performed in the three main OSs, macOS, Windows
and Linux.~
- ~[ ] Auto-update manual QA, from released version of component to new
version (see [tools/tuf/test](../tools/tuf/test/README.md)).~
2023-05-25 11:11:53 +00:00
|
|
|
Host: fleet.ResultHostData{
|
2021-09-10 19:26:39 +00:00
|
|
|
ID: 23,
|
|
|
|
Hostname: "somehostaaa",
|
Reduce size of `DistributedQueryResult` to improve live query performance (#11882)
This was found while working on #10957.
When running a live query, a lot of unused host data is stored in Redis
and sent on every live query result message via websockets. The frontend
and fleetctl just need `id`, `hostname` and `display_name`. (This
becomes worse every time we add new fields to the `Host` struct.)
Sample of one websocket message result when running `SELECT * from
osquery_info;`:
size in `main`: 2234 bytes
```
a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":57,\"host\":
{\"created_at\":\"2023-05-22T12:14:11Z\",\"updated_at\":\"2023-05-23T12:31:51Z\",
\"software_updated_at\":\"0001-01-01T00:00:00Z\",\"id\":106,\"detail_updated_at\":\"2023-05-23T11:50:04Z\",
\"label_updated_at\":\"2023-05-23T11:50:04Z\",\"policy_updated_at\":\"1970-01-02T00:00:00Z\",
\"last_enrolled_at\":\"2023-05-22T12:14:12Z\",
\"seen_time\":\"2023-05-23T09:52:23.876311-03:00\",\"refetch_requested\":false,
\"hostname\":\"lucass-macbook-pro.local\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"platform\":\"darwin\",\"osquery_version\":\"5.8.2\",\"os_version\":\"macOS 13.3.1\",\"build\":\"22E261\",\"platform_like\":\"darwin\",\"code_name\":\"\",
\"uptime\":91125000000000,\"memory\":34359738368,\"cpu_type\":\"x86_64h\",\"cpu_subtype\":\"Intel x86-64h Haswell\",\"cpu_brand\":\"Intel(R) Core(TM) i7-1068NG7 CPU @ 2.30GHz\",\"cpu_physical_cores\":4,\"cpu_logical_cores\":8,\"hardware_vendor\":\"Apple Inc.\",\"hardware_model\":\"MacBookPro16,2\",\"hardware_version\":\"1.0\",
\"hardware_serial\":\"0DPQR4HMD1FZ\",
\"computer_name\":\"Lucas’s MacBook Pro\",\"public_ip\":\"\",
\"primary_ip\":\"192.168.0.230\",\"primary_mac\":\"68:2f:67:8e:b6:1f\",
\"distributed_interval\":1,\"config_tls_refresh\":60,\"logger_tls_period\":10,\"team_id\":null,
\"pack_stats\":null,\"team_name\":null,
\"gigs_disk_space_available\":386.23,\"percent_disk_space_available\":40,
\"issues\":{\"total_issues_count\":0,\"failing_policies_count\":0},
\"mdm\":{\"enrollment_status\":null,\"server_url\":null,\"name\":\"\",\"encryption_key_available\":false},
\"status\":\"online\",\"display_text\":\"lucass-macbook-pro.local\",\"display_name\":\"Lucas’s MacBook Pro\"},
\"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\",
\"config_hash\":\"b7ee9363a7c686e76e99ffb122e9c5241a791e69\",\"config_valid\":\"1\",
\"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\",
\"host_hostname\":\"lucass-macbook-pro.local\",\"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\",\"start_time\":\"1684757652\",
\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",
\"version\":\"5.8.2\",\"watcher\":\"8364\"}],\"error\":null}}"]
```
vs. size of the message result on this branch: 675 bytes
```
a["{\"type\":\"result\",\"data\":{\"distributed_query_execution_id\":59,
\"host\":{\"id\":106,\"hostname\":\"lucass-macbook-pro.local\",
\"display_name\":\"Lucas’s MacBook Pro\"},
\"rows\":[{\"build_distro\":\"10.14\",\"build_platform\":\"darwin\",
\"config_hash\":\"f80dee827635db39077a458243379b3ad63311fd\",
\"config_valid\":\"1\",\"extensions\":\"active\",\"host_display_name\":\"Lucas’s MacBook Pro\",
\"host_hostname\":\"lucass-macbook-pro.local\",
\"instance_id\":\"cde5de81-344b-4c76-b1c5-dae964fdd4f2\",\"pid\":\"8370\",\"platform_mask\":\"21\",
\"start_time\":\"1684757652\",\"uuid\":\"BD4DFA10-E334-41D9-8136-D2163A8FE588\",\"version\":\"5.8.2\",
\"watcher\":\"8364\"}]}}"]
```
Manual tests included running with an old fleetctl running with a new
fleet server, and vice-versa, a new fleetctl running against an old
fleet server.
- [X] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [X] Documented any API changes (docs/Using-Fleet/REST-API.md or
docs/Contributing/API-for-contributors.md)
- ~[ ] Documented any permissions changes~
- ~[ ] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)~
- ~[ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for
new osquery data ingestion features.~
- [X] Added/updated tests
- [X] Manual QA for all new/changed functionality
- ~For Orbit and Fleet Desktop changes:~
- ~[ ] Manual QA must be performed in the three main OSs, macOS, Windows
and Linux.~
- ~[ ] Auto-update manual QA, from released version of component to new
version (see [tools/tuf/test](../tools/tuf/test/README.md)).~
2023-05-25 11:11:53 +00:00
|
|
|
},
|
2021-09-10 19:26:39 +00:00
|
|
|
Rows: []map[string]string{
|
|
|
|
{
|
|
|
|
"col1": "aaa",
|
|
|
|
"col2": "bbb",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Error: nil,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
b, err := json.Marshal(result)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
_ = ws.WriteMessage(mt, b)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}))
|
|
|
|
defer ts.Close()
|
|
|
|
|
|
|
|
baseURL, err := url.Parse(ts.URL)
|
|
|
|
require.NoError(t, err)
|
|
|
|
client := &Client{
|
2022-06-01 23:05:05 +00:00
|
|
|
baseClient: &baseClient{
|
|
|
|
baseURL: baseURL,
|
|
|
|
http: fleethttp.NewClient(),
|
|
|
|
insecureSkipVerify: false,
|
|
|
|
urlPrefix: "",
|
|
|
|
},
|
2023-06-15 17:13:41 +00:00
|
|
|
token: "1234",
|
|
|
|
outputWriter: nil,
|
2021-09-10 19:26:39 +00:00
|
|
|
}
|
|
|
|
ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Second)
|
|
|
|
defer cancelFunc()
|
|
|
|
|
2023-12-15 18:55:39 +00:00
|
|
|
res, err := client.LiveQueryWithContext(ctx, "select 1;", nil, nil, []string{"host1"})
|
2021-09-10 19:26:39 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
gotResults := false
|
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-res.Results():
|
|
|
|
gotResults = true
|
|
|
|
cancelFunc()
|
|
|
|
case err := <-res.Errors():
|
|
|
|
require.NoError(t, err)
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
<-ctx.Done()
|
|
|
|
assert.True(t, gotResults)
|
|
|
|
}
|