Add a setup guide for new CLI users (#1798)

This commit is contained in:
Mike Arpaia 2018-05-22 17:57:56 -06:00 committed by GitHub
parent 9924d07bb0
commit 1d70b082a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 461 additions and 259 deletions

View File

@ -36,6 +36,7 @@ func main() {
getPacksCommand(),
getLabelsCommand(),
getOptionsCommand(),
getEnrollSecretCommand(),
},
},
cli.Command{

View File

@ -264,7 +264,35 @@ func getOptionsCommand() cli.Command {
fmt.Print(string(b))
return nil
},
}
}
func getEnrollSecretCommand() cli.Command {
return cli.Command{
Name: "enroll-secret",
Usage: "Retrieve the osquery enroll secret",
Flags: []cli.Flag{
configFlag(),
contextFlag(),
},
Action: func(c *cli.Context) error {
fleet, err := clientFromCLI(c)
if err != nil {
return err
}
settings, err := fleet.GetServerSettings()
if err != nil {
return err
}
if settings == nil {
return errors.New("error: server setting were nil")
}
fmt.Println(*settings.EnrollSecret)
return nil
},
}
}

View File

@ -63,6 +63,7 @@ Interactively prompts for email and password if not specified in the flags or en
if err != nil {
return errors.Wrap(err, "error reading password")
}
fmt.Println()
flPassword = string(passBytes)
}

View File

@ -2,10 +2,12 @@ package main
import (
"fmt"
"os"
"github.com/kolide/fleet/server/service"
"github.com/pkg/errors"
"github.com/urfave/cli"
"golang.org/x/crypto/ssh/terminal"
)
func setupCommand() cli.Command {
@ -33,7 +35,7 @@ func setupCommand() cli.Command {
EnvVar: "PASSWORD",
Value: "",
Destination: &flPassword,
Usage: "Password for the admin user",
Usage: "Password for the admin user (recommended to use interactive entry)",
},
cli.StringFlag{
Name: "org-name",
@ -53,8 +55,15 @@ func setupCommand() cli.Command {
return errors.Errorf("Email of the admin user to create must be provided")
}
if flPassword == "" {
return errors.Errorf("Password for the admin user to create must be provided")
fmt.Print("Password: ")
passBytes, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
return errors.Wrap(err, "error reading password")
}
fmt.Println()
flPassword = string(passBytes)
}
token, err := fleet.Setup(flEmail, flPassword, flOrgName)
if err != nil {
switch err.(type) {

View File

@ -1,7 +1,12 @@
CLI Documentation
=================
Kolide Fleet provides a server which allows you to manage and orchestrate an osquery deployment across of a set of workstations and servers. For certain use-cases, it makes sense to maintain the configuration and data of an osquery deployment in source-controlled files. It is also desirable to be able to manage these files with a familiar command-line tool. To facilitate this, Kolide is working on an experimental CLI called `fleetctl`.
Kolide Fleet provides a server which allows you to manage and orchestrate an osquery deployment across of a set of workstations and servers. For certain use-cases, it makes sense to maintain the configuration and data of an osquery deployment in source-controlled files. It is also desirable to be able to manage these files with a familiar command-line tool. To facilitate this, Kolide Fleet includes a `fleetctl` CLI for managing osquery fleets in this way.
For more information, see:
- [Documentation on the file format](./file-format.md)
- [The setup guide for new CLI users](./setup-guide.md)
## Inspiration
@ -10,7 +15,7 @@ Inspiration for the `fleetctl` command-line experience as well as the file forma
- Format Familiarity: At Kolide, we love Kubernetes and we think it is the future of production infrastructure management. We believe that many of the people that use this interface to manage Fleet will also be Kubernetes operators. By using a familiar command-line interface and file format, the cognitive overhead can be reduced since the operator is already familiar with how these tools work and behave.
- Established Best Practices: Kubernetes deployments can easily become very complex. Because of this, Kubernetes operators have an established set of best practices that they often follow when writing and maintaining config files. Some of these best practices and tips are documented on the [official Kubernetes website](https://kubernetes.io/docs/concepts/configuration/overview/#general-config-tips) and some are documented by [the community](https://www.mirantis.com/blog/introduction-to-yaml-creating-a-kubernetes-deployment/). Since the file format and workflow is so similar, we can re-use these best practices when managing Fleet configurations.
## `fleetctl` - The CLI
### `fleetctl` - The CLI
The `fleetctl` tool is heavily inspired by the [`kubectl`](https://kubernetes.io/docs/user-guide/kubectl-overview/) tool. If you are familiar with `kubectl`, this will all feel very familiar to you. If not, some further explanation would likely be helpful.
@ -19,257 +24,3 @@ Fleet exposes the aspects of an osquery deployment as a set of "objects". Object
> Objects can be created, updated, and deleted by storing multiple object configuration files in a directory and using `kubectl apply` to recursively create and update those objects as needed.
Similarly, Fleet objects can be created, updated, and deleted by storing multiple object configuration files in a directory and using `fleetctl apply` to recursively create and update those objects as needed.
### Help Output
```
NAME:
fleetctl - The CLI for operating Kolide Fleet
USAGE:
fleetctl [global options] command [command options] [arguments...]
VERSION:
2.0.0-rc1
COMMANDS:
query Run an osquery distributed query
apply Apply files to declaratively manage osquery configurations
delete Specify files to delete
setup Setup a Kolide Fleet instance
login Login to Kolide Fleet
logout Logout of Kolide Fleet
get Get/list resources
config Modify how and which Fleet server to connect to
help, h Shows a list of commands or help for one command
GLOBAL OPTIONS:
--help, -h show help
--version, -v print the version
```
### Workflow
```bash
# Make sure you're currently using the current server (in this case: production linux hosts)
fleetctl config set-context production-linux
# Edit the config file (or files) for your Fleet instance (or one of them) and apply the file
vim fleet-linux.yml
fleetctl apply -f ./fleet-linux.yml
# Commit the changes to an upstream source tree
git add fleet-linux.yml
git commit -m "new changes to osquery production linux configuration"
git push
```
## Configuration File Format
A Fleet configuration is defined using one or more declarative "messages" in yaml syntax. Each message can live in it's own file or multiple in one file, each separated by `---`. Each file/message contains a few required top-level keys:
- `apiVersion` - the API version of the file/request
- `spec` - the "data" of the request
- `kind ` - the type of file/object (i.e.: pack, query, config)
The file may optionally also include some `metadata` for more complex data types (i.e.: packs).
When you reason about how to manage these config files, consider following the [General Config Tips](https://kubernetes.io/docs/concepts/configuration/overview/#general-config-tips) published by the Kubernetes project. Some of the especially relevant tips are included here as well:
- When defining configurations, specify the latest stable API version.
- Configuration files should be stored in version control before being pushed to the cluster. This allows quick roll-back of a configuration if needed. It also aids with cluster re-creation and restoration if necessary.
- Group related objects into a single file whenever it makes sense. One file is often easier to manage than several. See the [config-single-file.yml](../../examples/config-single-file.yml) file as an example of this syntax.
- Dont specify default values unnecessarily simple and minimal configs will reduce errors.
All of these files can be concatenated together into [one file](../../examples/config-single-file.yml) (seperated by `---`), or they can be in [individual files with a directory structure](../../examples/config-many-files) like the following:
```
|-- config.yml
|-- labels.yml
|-- packs
| `-- osquery-monitoring.yml
`-- queries.yml
```
### Osquery Configuration Options
The following file describes configuration options passed to the osquery instance. All other configuration data will be over-written by the application of this file.
```yaml
apiVersion: v1
kind: options
spec:
config:
options:
distributed_interval: 3
distributed_tls_max_attempts: 3
logger_plugin: tls
logger_tls_endpoint: /api/v1/osquery/log
logger_tls_period: 10
decorators:
load:
- "SELECT version FROM osquery_info"
- "SELECT uuid AS host_uuid FROM system_info"
always:
- "SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1"
interval:
3600: "SELECT total_seconds AS uptime FROM uptime"
overrides:
# Note configs in overrides take precedence over the default config defined
# under the config key above. With this config file, the base config would
# only be used for Windows hosts, while Mac and Linux hosts would pull
# these overrides.
platforms:
darwin:
options:
distributed_interval: 10
distributed_tls_max_attempts: 10
logger_plugin: tls
logger_tls_endpoint: /api/v1/osquery/log
logger_tls_period: 300
disable_tables: chrome_extensions
docker_socket: /var/run/docker.sock
file_paths:
users:
- /Users/%/Library/%%
- /Users/%/Documents/%%
etc:
- /etc/%%
linux:
options:
distributed_interval: 10
distributed_tls_max_attempts: 3
logger_plugin: tls
logger_tls_endpoint: /api/v1/osquery/log
logger_tls_period: 60
schedule_timeout: 60
docker_socket: /etc/run/docker.sock
file_paths:
homes:
- /root/.ssh/%%
- /home/%/.ssh/%%
etc:
- /etc/%%
tmp:
- /tmp/%%
exclude_paths:
homes:
- /home/not_to_monitor/.ssh/%%
tmp:
- /tmp/too_many_events/
decorators:
load:
- "SELECT * FROM cpuid"
- "SELECT * FROM docker_info"
interval:
3600: "SELECT total_seconds AS uptime FROM uptime"
```
### Host Labels
The following file describes the labels which hosts should be automatically grouped into. The label resource should reference the query by name. Both of these resources can be included in the same file as such:
```yaml
apiVersion: v1
kind: label
spec:
name: slack_not_running
query: slack_not_running
---
apiVersion: kolide.com/v1/alpha1
kind: query
spec:
name: slack_not_running
query: >
SELECT * from system_info
WHERE NOT EXISTS (
SELECT *
FROM processes
WHERE name LIKE "%Slack%"
);
```
### Osquery Queries
For especially long or complex queries, you may want to define one query in one file. Continued edits and applications to this file will update the query as long as the `metadata.name` does not change. If you want to change the name of a query, you must first create a new query with the new name and then delete the query with the old name. Make sure the old query name is not defined in any packs before deleting it or an error will occur.
```yaml
apiVersion: v1
kind: query
spec:
name: docker_processes
descriptions: The docker containers processes that are running on a system.
query: select * from docker_container_processes;
support:
osquery: 2.9.0
platforms:
- linux
- darwin
```
To define multiple queries in a file, concatenate multiple `query` resources together in a single file with `---`. For example, consider a file that you might store at `queries/osquery_monitoring.yml`:
```yaml
apiVersion: v1
kind: query
spec:
name: osquery_version
description: The version of the Launcher and Osquery process
query: select launcher.version, osquery.version from kolide_launcher_info launcher, osquery_info osquery;
support:
launcher: 0.3.0
osquery: 2.9.0
---
apiVersion: v1
kind: query
spec:
name: osquery_schedule
description: Report performance stats for each file in the query schedule.
query: select name, interval, executions, output_size, wall_time, (user_time/executions) as avg_user_time, (system_time/executions) as avg_system_time, average_memory, last_executed from osquery_schedule;
---
apiVersion: v1
kind: query
spec:
name: osquery_info
description: A heartbeat counter that reports general performance (CPU, memory) and version.
query: select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid;
---
apiVersion: v1
kind: query
spec:
name: osquery_events
description: Report event publisher health and track event counters.
query: select name, publisher, type, subscriptions, events, active from osquery_events;
```
### Query Packs
To define query packs, reference queries defined elsewhere by name. This is why the "name" of a query is so important. You can define many of these packs in many files.
```yaml
apiVersion: v1
kind: pack
spec:
name: osquery_monitoring
targets:
labels:
- all_hosts
queries:
- query: osquery_version
name: osquery_version_differential
interval: 7200
- query: osquery_version
name: osquery_version_snapshot
interval: 7200
snapshot: true
- query: osquery_schedule
interval: 7200
removed: false
- query: osquery_events
interval: 86400
removed: false
- query: oquery_info
interval: 600
removed: false
```

209
docs/cli/file-format.md Normal file
View File

@ -0,0 +1,209 @@
# Configuration File Format
A Fleet configuration is defined using one or more declarative "messages" in yaml syntax. Each message can live in it's own file or multiple in one file, each separated by `---`. Each file/message contains a few required top-level keys:
- `apiVersion` - the API version of the file/request
- `spec` - the "data" of the request
- `kind ` - the type of file/object (i.e.: pack, query, config)
The file may optionally also include some `metadata` for more complex data types (i.e.: packs).
When you reason about how to manage these config files, consider following the [General Config Tips](https://kubernetes.io/docs/concepts/configuration/overview/#general-config-tips) published by the Kubernetes project. Some of the especially relevant tips are included here as well:
- When defining configurations, specify the latest stable API version.
- Configuration files should be stored in version control before being pushed to the cluster. This allows quick roll-back of a configuration if needed. It also aids with cluster re-creation and restoration if necessary.
- Group related objects into a single file whenever it makes sense. One file is often easier to manage than several. See the [config-single-file.yml](../../examples/config-single-file.yml) file as an example of this syntax.
- Dont specify default values unnecessarily simple and minimal configs will reduce errors.
All of these files can be concatenated together into [one file](../../examples/config-single-file.yml) (seperated by `---`), or they can be in [individual files with a directory structure](../../examples/config-many-files) like the following:
```
|-- config.yml
|-- labels.yml
|-- packs
| `-- osquery-monitoring.yml
`-- queries.yml
```
## Osquery Configuration Options
The following file describes configuration options passed to the osquery instance. All other configuration data will be over-written by the application of this file.
```yaml
apiVersion: v1
kind: options
spec:
config:
options:
distributed_interval: 3
distributed_tls_max_attempts: 3
logger_plugin: tls
logger_tls_endpoint: /api/v1/osquery/log
logger_tls_period: 10
decorators:
load:
- "SELECT version FROM osquery_info"
- "SELECT uuid AS host_uuid FROM system_info"
always:
- "SELECT user AS username FROM logged_in_users WHERE user <> '' ORDER BY time LIMIT 1"
interval:
3600: "SELECT total_seconds AS uptime FROM uptime"
overrides:
# Note configs in overrides take precedence over the default config defined
# under the config key above. With this config file, the base config would
# only be used for Windows hosts, while Mac and Linux hosts would pull
# these overrides.
platforms:
darwin:
options:
distributed_interval: 10
distributed_tls_max_attempts: 10
logger_plugin: tls
logger_tls_endpoint: /api/v1/osquery/log
logger_tls_period: 300
disable_tables: chrome_extensions
docker_socket: /var/run/docker.sock
file_paths:
users:
- /Users/%/Library/%%
- /Users/%/Documents/%%
etc:
- /etc/%%
linux:
options:
distributed_interval: 10
distributed_tls_max_attempts: 3
logger_plugin: tls
logger_tls_endpoint: /api/v1/osquery/log
logger_tls_period: 60
schedule_timeout: 60
docker_socket: /etc/run/docker.sock
file_paths:
homes:
- /root/.ssh/%%
- /home/%/.ssh/%%
etc:
- /etc/%%
tmp:
- /tmp/%%
exclude_paths:
homes:
- /home/not_to_monitor/.ssh/%%
tmp:
- /tmp/too_many_events/
decorators:
load:
- "SELECT * FROM cpuid"
- "SELECT * FROM docker_info"
interval:
3600: "SELECT total_seconds AS uptime FROM uptime"
```
## Host Labels
The following file describes the labels which hosts should be automatically grouped into. The label resource should reference the query by name. Both of these resources can be included in the same file as such:
```yaml
apiVersion: v1
kind: label
spec:
name: slack_not_running
query: slack_not_running
---
apiVersion: kolide.com/v1/alpha1
kind: query
spec:
name: slack_not_running
query: >
SELECT * from system_info
WHERE NOT EXISTS (
SELECT *
FROM processes
WHERE name LIKE "%Slack%"
);
```
## Osquery Queries
For especially long or complex queries, you may want to define one query in one file. Continued edits and applications to this file will update the query as long as the `metadata.name` does not change. If you want to change the name of a query, you must first create a new query with the new name and then delete the query with the old name. Make sure the old query name is not defined in any packs before deleting it or an error will occur.
```yaml
apiVersion: v1
kind: query
spec:
name: docker_processes
descriptions: The docker containers processes that are running on a system.
query: select * from docker_container_processes;
support:
osquery: 2.9.0
platforms:
- linux
- darwin
```
To define multiple queries in a file, concatenate multiple `query` resources together in a single file with `---`. For example, consider a file that you might store at `queries/osquery_monitoring.yml`:
```yaml
apiVersion: v1
kind: query
spec:
name: osquery_version
description: The version of the Launcher and Osquery process
query: select launcher.version, osquery.version from kolide_launcher_info launcher, osquery_info osquery;
support:
launcher: 0.3.0
osquery: 2.9.0
---
apiVersion: v1
kind: query
spec:
name: osquery_schedule
description: Report performance stats for each file in the query schedule.
query: select name, interval, executions, output_size, wall_time, (user_time/executions) as avg_user_time, (system_time/executions) as avg_system_time, average_memory, last_executed from osquery_schedule;
---
apiVersion: v1
kind: query
spec:
name: osquery_info
description: A heartbeat counter that reports general performance (CPU, memory) and version.
query: select i.*, p.resident_size, p.user_time, p.system_time, time.minutes as counter from osquery_info i, processes p, time where p.pid = i.pid;
---
apiVersion: v1
kind: query
spec:
name: osquery_events
description: Report event publisher health and track event counters.
query: select name, publisher, type, subscriptions, events, active from osquery_events;
```
## Query Packs
To define query packs, reference queries defined elsewhere by name. This is why the "name" of a query is so important. You can define many of these packs in many files.
```yaml
apiVersion: v1
kind: pack
spec:
name: osquery_monitoring
targets:
labels:
- all_hosts
queries:
- query: osquery_version
name: osquery_version_differential
interval: 7200
- query: osquery_version
name: osquery_version_snapshot
interval: 7200
snapshot: true
- query: osquery_schedule
interval: 7200
removed: false
- query: osquery_events
interval: 86400
removed: false
- query: oquery_info
interval: 600
removed: false
```

165
docs/cli/setup-guide.md Normal file
View File

@ -0,0 +1,165 @@
# Setting Up Fleet via the CLI
In this document, I'm going to walk through how to setup and configure Kolide Fleet using just the CLI (which in-turn uses the Go API client). This document will hopefully illustrate:
- A minimal CLI workflow for managing an osquery fleet
- The set of API interactions that are required if you want to perform remote, automated management of a Fleet instance
## Running Fleet
For the sake of this tutorial, I will be using the local development Docker Compose infrastructure to run Fleet locally. This is documented in some detail in the [developer documentation](../development/development-infrastructure.md), but the following are the minimal set of commands that you can run from the root of the repository (assuming that you have a working Go/JavaScript toolchain installed along with Docker Compose):
```
docker-compose up -d
make deps
make generate
make
./build/fleet prepare db
./build/fleet serve --auth_jwt_key="insecure"
```
The `fleet serve` command will be the long running command that runs the Fleet server.
## `fleetctl config`
At this point, the MySQL database doesn't have any users in it. Because of this, Fleet is exposing a one-time setup endpoint. Before we can hit that endpoint (by running `fleetctl setup`), we have to first configure the local `fleetctl` context.
Now, since our Fleet instance is local in this tutorial, we didn't get a valid TLS certificate, so we need to run the following to configure our Fleet context:
```
$ fleetctl config set --address https://localhost:8080 --tls-skip-verify
[+] Set the address config key to "https://localhost:8080" in the "default" context
[+] Set the tls-skip-verify config key to "true" in the "default" context
```
Now, if you were connecting to a Fleet instance for real, you wouldn't want to skip TLS certificate verification, so you might run something like:
```
$ fleetctl config set --address https://fleet.osquery.tools
[+] Set the address config key to "https://fleet.osquery.tools" in the "default" context
```
## `fleetctl setup`
Now that we've configured our local CLI context, lets go ahead and create our admin account:
```
$ fleetctl setup --email mike@arpaia.co
Password:
[+] Fleet setup successful and context configured!
```
It's possible to specify the password via the `--password` flag or the `$PASSWORD` environment variable, but be cautious of the security implications of such an action. For local use, the interactive mode above is the most secure.
## Connecting a Host
For the sake of this tutorial, I'm going to be using Kolide's osquery launcher to start osquery locally and connect it to Fleet. To learn more about connecting osquery to Fleet, see the [Adding Hosts to Fleet](../infrastructure/adding-hosts-to-fleet.md) documentation.
To get your osquery enroll secret, run the following:
```
$ fleetctl get enroll-secret
E7P6zs9D0mvY7ct08weZ7xvLtQfGYrdC
```
You need to use this secret to connect a host. If you're running Fleet locally, you'd run:
```
launcher \
--hostname localhost:8080 \
--enroll_secret E7P6zs9D0mvY7ct08weZ7xvLtQfGYrdC \
--root_directory=$(mktemp -d) \
--insecure
```
## Query Hosts
To run a simple query against all hosts, you might run something like the following:
```
$ fleetctl query --query 'select * from osquery_info;' --labels='All Hosts' > results.json
⠂ 100% responded (100% online) | 1/1 targeted hosts (1/1 online)
^C
```
When the query is done (or you have enough results), CTRL-C and look at the `results.json` file:
```json
{
"host": "marpaia",
"rows": [
{
"build_distro": "10.13",
"build_platform": "darwin",
"config_hash": "d7cafcd183cc50c686b4c128263bd4eace5d89e1",
"config_valid": "1",
"extensions": "active",
"host_hostname": "marpaia",
"instance_id": "37840766-7182-4a68-a204-c7f577bd71e1",
"pid": "22984",
"start_time": "1527031727",
"uuid": "B312055D-9209-5C89-9DDB-987299518FF7",
"version": "3.2.3",
"watcher": "-1"
}
]
}
```
## Update Osquery Options
By default, each osquery node will check in with Fleet every 10 seconds. Let's say, for testing, you want to increase this to every 2 seconds. If this is the first time you've ever modified osquery options, let's download them locally:
```
fleetctl get options > options.yaml
```
The `options.yaml` file will look something like this:
```yaml
apiVersion: v1
kind: options
spec:
config:
decorators:
load:
- SELECT uuid AS host_uuid FROM system_info;
- SELECT hostname AS hostname FROM system_info;
options:
disable_distributed: false
distributed_interval: 10
distributed_plugin: tls
distributed_tls_max_attempts: 3
distributed_tls_read_endpoint: /api/v1/osquery/distributed/read
distributed_tls_write_endpoint: /api/v1/osquery/distributed/write
logger_plugin: tls
logger_tls_endpoint: /api/v1/osquery/log
logger_tls_period: 10
pack_delimiter: /
overrides: {}
```
Let's edit the file so that the `distributed_interval` option is 2 instead of 10. Save the file and run:
```
fleetctl apply -f ./options.yaml
```
Now run a live query again. You should notice results coming back more quickly.
## Logging In To An Existing Fleet Instance
If you have an existing Fleet instance (version 2.0.0 or above), then simply run `fleet login` (after configuring your local CLI context):
```
$ fleetctl config set --address https://fleet.osquery.tools
[+] Set the address config key to "https://fleet.osquery.tools" in the "default" context
$ fleetctl login
Log in using the standard Fleet credentials.
Email: mike@arpaia.co
Password:
[+] Fleet login successful and context configured!
```
Once your local context is configured, you can use the above `fleetctl` normally. See `fleetctl --help` for more information.

View File

@ -0,0 +1,38 @@
package service
import (
"encoding/json"
"net/http"
"github.com/kolide/fleet/server/kolide"
"github.com/pkg/errors"
)
// GetServerSettings fetches the server settings from the server API
func (c *Client) GetServerSettings() (*kolide.ServerSettings, error) {
response, err := c.AuthenticatedDo("GET", "/api/v1/kolide/config", nil)
if err != nil {
return nil, errors.Wrap(err, "GET /api/v1/kolide/config")
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, errors.Errorf(
"get server settings received status %d %s",
response.StatusCode,
extractServerErrorText(response.Body),
)
}
var responseBody appConfigResponse
err = json.NewDecoder(response.Body).Decode(&responseBody)
if err != nil {
return nil, errors.Wrap(err, "decode get app config response")
}
if responseBody.Err != nil {
return nil, errors.Errorf("get server settings: %s", responseBody.Err)
}
return responseBody.ServerSettings, nil
}