empayre/fleet
14
0
Fork 0
mirror of https://github.com/empayre/fleet.git synced 2024-11-06 08:55:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add osquery perf (#2190)

Browse Source 
* Add osquery perf

* Update dockerfile and gh action
This commit is contained in:
Tomas Touceda 2021-09-22 17:18:55 -03:00 committed by GitHub
parent 3ea0439cf0
commit 8600d71d35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 734 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
.github/workflows/push-osquery-perf-to-ecr.yml vendored Normal file
View File

@ -0,0 +1,39 @@
name: Build docker image and publish to ECR
on:
workflow_dispatch:
inputs:
enroll_secret:
description: 'Enroll Secret'
required: true
url:
description: 'Fleet server URL'
required: true
tag:
description: 'docker image tag'
required: true
default: latest
jobs:
build-docker:
runs-on: ubuntu-latest
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.LOADTEST_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.LOADTEST_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Build, tag, and push image to Amazon ECR
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: osquery-perf
IMAGE_TAG: ${{ github.event.inputs.tag }}
run: |
docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG --build-arg ENROLL_SECRET=${{ github.event.inputs.enroll_secret }} --build-arg HOST_COUNT=${{ github.event.inputs.host_count }} --build-arg SERVER_URL=${{ github.event.inputs.url }} -f Dockerfile.osquery-perf .
docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

16
Dockerfile.osquery-perf Normal file
View File

@ -0,0 +1,16 @@
FROM golang:1.17.1-alpine
ARG ENROLL_SECRET
ARG HOST_COUNT
ARG SERVER_URL
ENV ENROLL_SECRET ${ENROLL_SECRET}
ENV HOST_COUNT ${HOST_COUNT}
ENV SERVER_URL ${SERVER_URL}
COPY ./cmd/osquery-perf/agent.go ./go.mod ./go.sum ./cmd/osquery-perf/mac10.14.6.tmpl /osquery-perf/
WORKDIR /osquery-perf/
RUN go mod download
RUN go build -o osquery-perf
CMD ./osquery-perf -enroll_secret $ENROLL_SECRET -host_count $HOST_COUNT -server_url $SERVER_URL

82
cmd/osquery-perf/README.md Normal file
View File

@ -0,0 +1,82 @@
# Osquery Server Performance Tester
> **TODO: Archive this repo and move its contents inline into https://github.com/fleetdm/fleet**
This repository provides a tool to generate realistic traffic to an osquery
management server (primarily, [Fleet](https://github.com/fleetdm/fleet)). With
this tool, many thousands of hosts can be simulated from a single host.
## Requirements
The only requirement for running this tool is a working installation of
[Go](https://golang.org/doc/install).
## Usage
Typically `go run` is used:
```
go run agent.go --help
Usage of agent.go:
-config_interval duration
Interval for config requests (default 1m0s)
-enroll_secret string
Enroll secret to authenticate enrollment
-host_count int
Number of hosts to start (default 10) (default 10)
-query_interval duration
Interval for live query requests (default 10s)
-seed int
Seed for random generator (default current time) (default 1586310930917739000)
-server_url string
URL (with protocol and port of osquery server) (default "https://localhost:8080")
-start_period duration
Duration to spread start of hosts over (default 10s)
exit status 2
```
The tool should be invoked with the appropriate enroll secret. A typical
invocation looks like:
```
go run agent.go --enroll_secret hgh4hk3434l2jjf
```
When starting many hosts, it is a good idea to extend the intervals, and also
the period over which the hosts are started:
```
go run agent.go --enroll_secret hgh4hk3434l2jjf --host_count 5000 --start_period 5m --query_interval 60s --config_interval 5m
```
This will start 5,000 hosts over a period of 5 minutes. Each host will check in
for live queries at a 1 minute interval, and for configuration at a 5 minute
interval. Starting over a 5 minute period ensures that the configuration
requests are spread evenly over the 5 minute interval.
It can be useful to start the "same" hosts. This can be achieved with the
`--seed` parameter:
```
go run agent.go --enroll_secret hgh4hk3434l2jjf --seed 0
```
By using the same seed, along with other values, we usually get hosts that look
the same to the server. This is not guaranteed, but it is a useful technique.
### Resource Limits
On many systems, trying to simulate a large number of hosts will result in hitting system resource limits (such as number of open file descriptors).
If you see errors such as `dial tcp: lookup localhost: no such host` or `read: connection reset by peer`, try increasing these limits.
#### macOS
Run the following command in the shell before running the Fleet server _and_ before running `agent.go` (run it once in each shell):
``` sh
ulimit -n 64000
```
## Bugs
To report a bug, [click here](https://github.com/fleetdm/fleet).

285
cmd/osquery-perf/agent.go Normal file
View File

@ -0,0 +1,285 @@
package main
import (
"bytes"
"crypto/tls"
"encoding/json"
"flag"
"fmt"
"log"
"math/rand"
"net/http"
"strings"
"text/template"
"time"
"github.com/google/uuid"
)
type Agent struct {
ServerAddress string
EnrollSecret string
NodeKey string
UUID string
Client http.Client
ConfigInterval time.Duration
QueryInterval time.Duration
Templates *template.Template
strings map[string]string
}
func NewAgent(serverAddress, enrollSecret string, templates *template.Template, configInterval, queryInterval time.Duration) *Agent {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
transport.DisableCompression = true
return &Agent{
ServerAddress: serverAddress,
EnrollSecret: enrollSecret,
Templates: templates,
ConfigInterval: configInterval,
QueryInterval: queryInterval,
UUID: uuid.New().String(),
Client: http.Client{Transport: transport},
strings: make(map[string]string),
}
}
type enrollResponse struct {
NodeKey string `json:"node_key"`
}
type distributedReadResponse struct {
Queries map[string]string `json:"queries"`
}
func (a *Agent) runLoop() {
a.Enroll()
a.Config()
resp, err := a.DistributedRead()
if err != nil {
log.Println(err)
} else {
if len(resp.Queries) > 0 {
a.DistributedWrite(resp.Queries)
}
}
configTicker := time.Tick(a.ConfigInterval)
liveQueryTicker := time.Tick(a.QueryInterval)
for {
select {
case <-configTicker:
a.Config()
case <-liveQueryTicker:
resp, err := a.DistributedRead()
if err != nil {
log.Println(err)
} else {
if len(resp.Queries) > 0 {
a.DistributedWrite(resp.Queries)
}
}
}
}
}
const stringVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_."
func (a *Agent) randomString(n int) string {
sb := strings.Builder{}
sb.Grow(n)
for i := 0; i < n; i++ {
sb.WriteByte(stringVals[rand.Int63()%int64(len(stringVals))])
}
return sb.String()
}
func (a *Agent) CachedString(key string) string {
if val, ok := a.strings[key]; ok {
return val
}
val := a.randomString(12)
a.strings[key] = val
return val
}
func (a *Agent) Enroll() {
var body bytes.Buffer
if err := a.Templates.ExecuteTemplate(&body, "enroll", a); err != nil {
log.Println("execute template:", err)
return
}
req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/enroll", &body)
if err != nil {
log.Println("create request:", err)
return
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", "osquery/4.6.0")
resp, err := a.Client.Do(req)
if err != nil {
log.Println("do request:", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Println("status:", resp.Status)
return
}
var parsedResp enrollResponse
if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil {
log.Println("json parse:", err)
return
}
a.NodeKey = parsedResp.NodeKey
}
func (a *Agent) Config() {
body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`)
req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/config", body)
if err != nil {
log.Println("create config request:", err)
return
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", "osquery/4.6.0")
resp, err := a.Client.Do(req)
if err != nil {
log.Println("do config request:", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Println("config status:", resp.Status)
return
}
// No need to read the config body
}
func (a *Agent) DistributedRead() (*distributedReadResponse, error) {
body := bytes.NewBufferString(`{"node_key": "` + a.NodeKey + `"}`)
req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/read", body)
if err != nil {
return nil, fmt.Errorf("create distributed read request: %s", err)
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", "osquery/4.6.0")
resp, err := a.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("do distributed read request: %s", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("distributed read status: %s", resp.Status)
}
var parsedResp distributedReadResponse
if err := json.NewDecoder(resp.Body).Decode(&parsedResp); err != nil {
return nil, fmt.Errorf("json parse distributed read response: %s", err)
}
return &parsedResp, nil
}
type distributedWriteRequest struct {
Queries map[string]json.RawMessage `json:"queries"`
Statuses map[string]string `json:"statuses"`
NodeKey string `json:"node_key"`
}
var defaultQueryResult = json.RawMessage(`[{"foo": "bar"}]`)
const statusSuccess = "0"
func (a *Agent) DistributedWrite(queries map[string]string) {
var body bytes.Buffer
if _, ok := queries["fleet_detail_query_network_interface"]; ok {
// Respond to label/detail queries
a.Templates.ExecuteTemplate(&body, "distributed_write", a)
} else {
// Return a generic response for any other queries
req := distributedWriteRequest{
Queries: make(map[string]json.RawMessage),
Statuses: make(map[string]string),
NodeKey: a.NodeKey,
}
for name := range queries {
req.Queries[name] = defaultQueryResult
req.Statuses[name] = statusSuccess
}
json.NewEncoder(&body).Encode(req)
}
req, err := http.NewRequest("POST", a.ServerAddress+"/api/v1/osquery/distributed/write", &body)
if err != nil {
log.Println("create distributed write request:", err)
return
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Accept", "application/json")
req.Header.Add("User-Agent", "osquery/4.6.0")
resp, err := a.Client.Do(req)
if err != nil {
log.Println("do distributed write request:", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Println("distributed write status:", resp.Status)
return
}
// No need to read the distributed write body
}
func main() {
serverURL := flag.String("server_url", "https://localhost:8080", "URL (with protocol and port of osquery server)")
enrollSecret := flag.String("enroll_secret", "", "Enroll secret to authenticate enrollment")
hostCount := flag.Int("host_count", 10, "Number of hosts to start (default 10)")
randSeed := flag.Int64("seed", time.Now().UnixNano(), "Seed for random generator (default current time)")
startPeriod := flag.Duration("start_period", 10*time.Second, "Duration to spread start of hosts over")
configInterval := flag.Duration("config_interval", 1*time.Minute, "Interval for config requests")
queryInterval := flag.Duration("query_interval", 10*time.Second, "Interval for live query requests")
flag.Parse()
rand.Seed(*randSeed)
tmpl, err := template.ParseGlob("*.tmpl")
if err != nil {
log.Fatal("parse templates: ", err)
}
// Spread starts over the interval to prevent thunering herd
sleepTime := *startPeriod / time.Duration(*hostCount)
var agents []*Agent
for i := 0; i < *hostCount; i++ {
a := NewAgent(*serverURL, *enrollSecret, tmpl, *configInterval, *queryInterval)
agents = append(agents, a)
go a.runLoop()
time.Sleep(sleepTime)
}
fmt.Println("Agents running. Kill with C-c.")
<-make(chan struct{})
}

309
cmd/osquery-perf/mac10.14.6.tmpl Normal file
View File

@ -0,0 +1,309 @@
{{ define "enroll" -}}
{
"enroll_secret": "{{ .EnrollSecret }}",
"host_details": {
"os_version": {
"build": "18G3020",
"major": "10",
"minor": "14",
"name": "Mac OS X",
"patch": "6",
"platform": "darwin",
"platform_like": "darwin",
"version": "10.14.6"
},
"osquery_info": {
"build_distro": "10.12",
"build_platform": "darwin",
"config_hash": "",
"config_valid": "0",
"extensions": "inactive",
"instance_id": "{{ .UUID }}",
"pid": "12947",
"platform_mask": "21",
"start_time": "1580931224",
"uuid": "{{ .UUID }}",
"version": "4.6.0",
"watcher": "12946"
},
"platform_info": {
"address": "0xff990000",
"date": "12/16/2019 ",
"extra": "MBP114; 196.0.0.0.0; root@xapp160; Mon Dec 16 15:55:18 PST 2019; 196 (B&I); F000_B00; Official Build, Release; Apple LLVM version 5.0 (clang-500.0.68) (based on LLVM 3.3svn)",
"revision": "196 (B&I)",
"size": "8388608",
"vendor": "Apple Inc. ",
"version": "196.0.0.0.0 ",
"volume_size": "1507328"
},
"system_info": {
"computer_name": "{{ .CachedString "hostname" }}",
"cpu_brand": "Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz\u0000\u0000\u0000\u0000\u0000\u0000\u0000",
"cpu_logical_cores": "8",
"cpu_physical_cores": "4",
"cpu_subtype": "Intel x86-64h Haswell",
"cpu_type": "x86_64h",
"hardware_model": "MacBookPro11,4",
"hardware_serial": "D02R835DG8WK",
"hardware_vendor": "Apple Inc.",
"hardware_version": "1.0",
"hostname": "{{ .CachedString "hostname" }}",
"local_hostname": "{{ .CachedString "hostname" }}",
"physical_memory": "17179869184",
"uuid": "{{ .UUID }}"
}
},
"host_identifier": "{{ .CachedString "hostname" }}",
"platform_type": "21"
}
{{- end }}
{{ define "distributed_write" -}}
{
"queries":{
"fleet_detail_query_network_interface":[
{
"point_to_point":"",
"address":"fe80::8cb:112d:ff51:1e5d%en0",
"mask":"ffff:ffff:ffff:ffff::",
"broadcast":"",
"interface":"en0",
"mac":"f8:2d:88:93:56:5c",
"type":"6",
"mtu":"1500",
"metric":"0",
"ipackets":"278493",
"opackets":"206238",
"ibytes":"275799040",
"obytes":"37720064",
"ierrors":"0",
"oerrors":"0",
"idrops":"0",
"odrops":"0",
"last_change":"1582848084"
},
{
"point_to_point":"",
"address":"192.168.1.3",
"mask":"255.255.255.0",
"broadcast":"192.168.1.255",
"interface":"en0",
"mac":"f5:5a:80:92:52:5b",
"type":"6",
"mtu":"1500",
"metric":"0",
"ipackets":"278493",
"opackets":"206238",
"ibytes":"275799040",
"obytes":"37720064",
"ierrors":"0",
"oerrors":"0",
"idrops":"0",
"odrops":"0",
"last_change":"1582848084"
},
{
"point_to_point":"127.0.0.1",
"address":"127.0.0.1",
"mask":"255.0.0.0",
"broadcast":"",
"interface":"lo0",
"mac":"00:00:00:00:00:00",
"type":"24",
"mtu":"16384",
"metric":"0",
"ipackets":"132952",
"opackets":"132952",
"ibytes":"67053568",
"obytes":"67053568",
"ierrors":"0",
"oerrors":"0",
"idrops":"0",
"odrops":"0",
"last_change":"1582840871"
},
{
"point_to_point":"::1",
"address":"::1",
"mask":"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
"broadcast":"",
"interface":"lo0",
"mac":"00:00:00:00:00:00",
"type":"24",
"mtu":"16384",
"metric":"0",
"ipackets":"132952",
"opackets":"132952",
"ibytes":"67053568",
"obytes":"67053568",
"ierrors":"0",
"oerrors":"0",
"idrops":"0",
"odrops":"0",
"last_change":"1582840871"
},
{
"point_to_point":"",
"address":"fe80::1%lo0",
"mask":"ffff:ffff:ffff:ffff::",
"broadcast":"",
"interface":"lo0",
"mac":"00:00:00:00:00:00",
"type":"24",
"mtu":"16384",
"metric":"0",
"ipackets":"132952",
"opackets":"132952",
"ibytes":"67053568",
"obytes":"67053568",
"ierrors":"0",
"oerrors":"0",
"idrops":"0",
"odrops":"0",
"last_change":"1582840871"
},
{
"point_to_point":"",
"address":"fe80::3a:84ff:fe6b:bf75%awdl0",
"mask":"ffff:ffff:ffff:ffff::",
"broadcast":"",
"interface":"awdl0",
"mac":"03:3b:94:5b:be:75",
"type":"6",
"mtu":"1484",
"metric":"0",
"ipackets":"0",
"opackets":"16",
"ibytes":"0",
"obytes":"3072",
"ierrors":"0",
"oerrors":"0",
"idrops":"0",
"odrops":"0",
"last_change":"1582842892"
},
{
"point_to_point":"",
"address":"fe80::6eaf:9721:3476:b691%utun0",
"mask":"ffff:ffff:ffff:ffff::",
"broadcast":"",
"interface":"utun0",
"mac":"00:00:00:00:00:00",
"type":"1",
"mtu":"2000",
"metric":"0",
"ipackets":"0",
"opackets":"2",
"ibytes":"0",
"obytes":"0",
"ierrors":"0",
"oerrors":"0",
"idrops":"0",
"odrops":"0",
"last_change":"1582840897"
}
],
"fleet_detail_query_os_version":[
{
"name":"Mac OS X",
"version":"10.14.6",
"major":"10",
"minor":"14",
"patch":"6",
"build":"18G3020",
"platform":"darwin",
"platform_like":"darwin",
"codename":""
}
],
"fleet_detail_query_osquery_flags":[
{
"name":"config_refresh",
"value":"{{ printf "%.0f" .ConfigInterval.Seconds }}"
},
{
"name":"distributed_interval",
"value":"{{ printf "%.0f" .QueryInterval.Seconds }}"
},
{
"name":"logger_tls_period",
"value":"99999"
}
],
"fleet_detail_query_osquery_info":[
{
"pid":"11287",
"uuid":"{{ .UUID }}",
"instance_id":"{{ .UUID }}",
"version":"4.1.2",
"config_hash":"b01efbf375ac6767f259ae98751154fef727ce35",
"config_valid":"1",
"extensions":"inactive",
"build_platform":"darwin",
"build_distro":"10.12",
"start_time":"1582857555",
"watcher":"11286",
"platform_mask":"21"
}
],
"fleet_detail_query_system_info":[
{
"hostname":"{{ .CachedString "hostname" }}",
"uuid":"4740D59F-699E-5B29-960B-979AAF9BBEEB",
"cpu_type":"x86_64h",
"cpu_subtype":"Intel x86-64h Haswell",
"cpu_brand":"Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz",
"cpu_physical_cores":"4",
"cpu_logical_cores":"8",
"cpu_microcode":"",
"physical_memory":"17179869184",
"hardware_vendor":"Apple Inc.",
"hardware_model":"MacBookPro11,4",
"hardware_version":"1.0",
"hardware_serial":"C02R262BM8LN",
"computer_name":"{{ .CachedString "hostname" }}",
"local_hostname":"{{ .CachedString "hostname" }}"
}
],
"fleet_detail_query_uptime":[
{
"days":"0",
"hours":"4",
"minutes":"38",
"seconds":"11",
"total_seconds":"16691"
}
]
},
"statuses":{
"fleet_detail_query_network_interface":0,
"fleet_detail_query_os_version":0,
"fleet_detail_query_osquery_flags":0,
"fleet_detail_query_osquery_info":0,
"fleet_detail_query_system_info":0,
"fleet_detail_query_uptime":0
},
"node_key":"{{ .NodeKey }}"
}
{{- end }}

2
go.mod
View File

@ -33,7 +33,7 @@ require (
github.com/gomodule/redigo v1.8.5
github.com/google/go-cmp v0.5.6
github.com/google/go-github/v37 v37.0.0
github.com/google/uuid v1.1.2
github.com/google/uuid v1.3.0
github.com/goreleaser/nfpm/v2 v2.2.2
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.4.2

2
go.sum
View File

@ -454,6 +454,8 @@ github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4Mgqvf
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=