2016-11-16 13:47:49 +00:00
package mysql
import (
"database/sql"
2016-12-01 17:00:00 +00:00
"fmt"
2021-05-07 04:05:09 +00:00
"strings"
2016-11-16 13:47:49 +00:00
"time"
2020-12-10 19:04:58 +00:00
"github.com/cenkalti/backoff/v4"
2021-06-26 04:46:51 +00:00
"github.com/fleetdm/fleet/v4/server/fleet"
2020-12-10 19:04:58 +00:00
"github.com/jmoiron/sqlx"
2017-01-04 21:16:17 +00:00
"github.com/pkg/errors"
2016-11-16 13:47:49 +00:00
)
2021-06-24 00:32:19 +00:00
var hostSearchColumns = [ ] string { "hostname" , "uuid" , "hardware_serial" , "primary_ip" }
2021-02-18 20:52:43 +00:00
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) NewHost ( host * fleet . Host ) ( * fleet . Host , error ) {
2016-11-16 13:47:49 +00:00
sqlStatement := `
INSERT INTO hosts (
2016-12-06 19:51:11 +00:00
osquery_host_id ,
2021-06-24 00:32:19 +00:00
detail_updated_at ,
label_updated_at ,
2016-11-16 13:47:49 +00:00
node_key ,
2021-06-24 00:32:19 +00:00
hostname ,
2016-11-16 13:47:49 +00:00
uuid ,
platform ,
osquery_version ,
os_version ,
uptime ,
2021-06-24 00:32:19 +00:00
memory ,
2021-05-27 20:18:00 +00:00
seen_time ,
team_id
2016-11-16 13:47:49 +00:00
)
2021-05-27 20:18:00 +00:00
VALUES ( ? , ? , ? , ? , ? , ? , ? , ? , ? , ? , ? , ? , ? )
2016-11-16 13:47:49 +00:00
`
2020-04-07 01:10:20 +00:00
result , err := d . db . Exec (
sqlStatement ,
host . OsqueryHostID ,
2021-06-24 00:32:19 +00:00
host . DetailUpdatedAt ,
host . LabelUpdatedAt ,
2020-04-07 01:10:20 +00:00
host . NodeKey ,
2021-06-24 00:32:19 +00:00
host . Hostname ,
2020-04-07 01:10:20 +00:00
host . UUID ,
host . Platform ,
host . OsqueryVersion ,
host . OSVersion ,
host . Uptime ,
2021-06-24 00:32:19 +00:00
host . Memory ,
2020-04-07 01:10:20 +00:00
host . SeenTime ,
2021-05-27 20:18:00 +00:00
host . TeamID ,
2020-04-07 01:10:20 +00:00
)
2016-11-16 13:47:49 +00:00
if err != nil {
2017-01-04 21:16:17 +00:00
return nil , errors . Wrap ( err , "new host" )
2016-11-16 13:47:49 +00:00
}
id , _ := result . LastInsertId ( )
host . ID = uint ( id )
return host , nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) SaveHost ( host * fleet . Host ) error {
2016-11-16 13:47:49 +00:00
sqlStatement := `
UPDATE hosts SET
2021-06-24 00:32:19 +00:00
detail_updated_at = ? ,
label_updated_at = ? ,
2016-11-16 13:47:49 +00:00
node_key = ? ,
2021-06-24 00:32:19 +00:00
hostname = ? ,
2016-11-16 13:47:49 +00:00
uuid = ? ,
platform = ? ,
osquery_version = ? ,
os_version = ? ,
uptime = ? ,
2021-06-24 00:32:19 +00:00
memory = ? ,
2016-12-01 17:00:00 +00:00
cpu_type = ? ,
cpu_subtype = ? ,
cpu_brand = ? ,
cpu_physical_cores = ? ,
hardware_vendor = ? ,
hardware_model = ? ,
hardware_version = ? ,
hardware_serial = ? ,
computer_name = ? ,
build = ? ,
platform_like = ? ,
code_name = ? ,
2017-01-04 21:16:17 +00:00
cpu_logical_cores = ? ,
2017-04-06 18:55:24 +00:00
seen_time = ? ,
distributed_interval = ? ,
config_tls_refresh = ? ,
2020-05-21 15:36:00 +00:00
logger_tls_period = ? ,
2021-05-31 16:02:05 +00:00
team_id = ? ,
2020-03-11 20:38:44 +00:00
primary_ip = ? ,
2021-05-13 20:09:22 +00:00
primary_mac = ? ,
refetch_requested = ?
2016-11-16 13:47:49 +00:00
WHERE id = ?
`
2020-03-11 20:38:44 +00:00
_ , err := d . db . Exec ( sqlStatement ,
2021-06-24 00:32:19 +00:00
host . DetailUpdatedAt ,
host . LabelUpdatedAt ,
2020-03-11 20:38:44 +00:00
host . NodeKey ,
2021-06-24 00:32:19 +00:00
host . Hostname ,
2020-03-11 20:38:44 +00:00
host . UUID ,
host . Platform ,
host . OsqueryVersion ,
host . OSVersion ,
host . Uptime ,
2021-06-24 00:32:19 +00:00
host . Memory ,
2020-03-11 20:38:44 +00:00
host . CPUType ,
host . CPUSubtype ,
host . CPUBrand ,
host . CPUPhysicalCores ,
host . HardwareVendor ,
host . HardwareModel ,
host . HardwareVersion ,
host . HardwareSerial ,
host . ComputerName ,
host . Build ,
host . PlatformLike ,
host . CodeName ,
host . CPULogicalCores ,
host . SeenTime ,
host . DistributedInterval ,
host . ConfigTLSRefresh ,
host . LoggerTLSPeriod ,
2021-05-31 16:02:05 +00:00
host . TeamID ,
2020-03-11 20:38:44 +00:00
host . PrimaryIP ,
host . PrimaryMac ,
2021-05-13 20:09:22 +00:00
host . RefetchRequested ,
2020-03-11 20:38:44 +00:00
host . ID ,
)
if err != nil {
return errors . Wrapf ( err , "save host with id %d" , host . ID )
}
2020-03-11 01:14:02 +00:00
2021-05-07 04:05:09 +00:00
// Save host pack stats only if it is non-nil. Empty stats should be
// represented by an empty slice.
if host . PackStats != nil {
if err := d . saveHostPackStats ( host ) ; err != nil {
return err
}
}
2021-07-13 20:15:38 +00:00
if host . HostSoftware . Modified {
if err := d . SaveHostSoftware ( host ) ; err != nil {
return errors . Wrap ( err , "failed to save host software" )
}
}
if host . Modified {
if err := d . SaveHostAdditional ( host ) ; err != nil {
return errors . Wrap ( err , "failed to save host additional" )
}
if err := d . SaveHostUsers ( host ) ; err != nil {
return errors . Wrap ( err , "failed to save host users" )
}
}
host . Modified = false
2021-05-07 04:05:09 +00:00
return nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) saveHostPackStats ( host * fleet . Host ) error {
2021-05-07 04:05:09 +00:00
if err := d . withRetryTxx ( func ( tx * sqlx . Tx ) error {
sql := `
DELETE FROM scheduled_query_stats
WHERE host_id = ?
`
if _ , err := tx . Exec ( sql , host . ID ) ; err != nil {
return errors . Wrap ( err , "delete old stats" )
}
// Bulk insert software entries
var args [ ] interface { }
queryCount := 0
for _ , pack := range host . PackStats {
for _ , query := range pack . QueryStats {
queryCount ++
args = append ( args ,
query . PackName ,
query . ScheduledQueryName ,
host . ID ,
query . AverageMemory ,
query . Denylisted ,
query . Executions ,
query . Interval ,
query . LastExecuted ,
query . OutputSize ,
query . SystemTime ,
query . UserTime ,
query . WallTime ,
)
}
}
if queryCount == 0 {
return nil
}
values := strings . TrimSuffix ( strings . Repeat ( "((SELECT sq.id FROM scheduled_queries sq JOIN packs p ON (sq.pack_id = p.id) WHERE p.name = ? AND sq.name = ?),?,?,?,?,?,?,?,?,?,?)," , queryCount ) , "," )
sql = fmt . Sprintf ( `
INSERT IGNORE INTO scheduled_query_stats (
scheduled_query_id ,
host_id ,
average_memory ,
denylisted ,
executions ,
schedule_interval ,
last_executed ,
output_size ,
system_time ,
user_time ,
wall_time
)
VALUES % s
` , values )
if _ , err := tx . Exec ( sql , args ... ) ; err != nil {
return errors . Wrap ( err , "insert pack stats" )
}
return nil
} ) ; err != nil {
return errors . Wrap ( err , "save pack stats" )
}
return nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) loadHostPackStats ( host * fleet . Host ) error {
2021-05-07 04:05:09 +00:00
sql := `
SELECT
sqs . scheduled_query_id ,
sqs . average_memory ,
sqs . denylisted ,
sqs . executions ,
sqs . schedule_interval ,
sqs . last_executed ,
sqs . output_size ,
sqs . system_time ,
sqs . user_time ,
sqs . wall_time ,
sq . name AS scheduled_query_name ,
sq . id AS scheduled_query_id ,
sq . query_name AS query_name ,
p . name AS pack_name ,
2021-05-07 19:47:52 +00:00
p . id as pack_id ,
q . description
2021-05-07 04:05:09 +00:00
FROM scheduled_query_stats sqs
JOIN scheduled_queries sq ON ( sqs . scheduled_query_id = sq . id )
2021-05-07 19:47:52 +00:00
JOIN packs p ON ( sq . pack_id = p . id )
JOIN queries q ON ( sq . query_name = q . name )
2021-05-07 04:05:09 +00:00
WHERE host_id = ?
`
2021-06-06 22:07:29 +00:00
var stats [ ] fleet . ScheduledQueryStats
2021-05-07 04:05:09 +00:00
if err := d . db . Select ( & stats , sql , host . ID ) ; err != nil {
return errors . Wrap ( err , "load pack stats" )
}
2021-06-06 22:07:29 +00:00
packs := map [ uint ] fleet . PackStats { }
2021-05-07 04:05:09 +00:00
for _ , query := range stats {
pack := packs [ query . PackID ]
pack . PackName = query . PackName
pack . PackID = query . PackID
pack . QueryStats = append ( pack . QueryStats , query )
packs [ pack . PackID ] = pack
}
for _ , pack := range packs {
host . PackStats = append ( host . PackStats , pack )
}
2020-03-11 20:38:44 +00:00
return nil
2016-11-16 13:47:49 +00:00
}
2021-07-13 20:15:38 +00:00
func ( d * Datastore ) loadHostUsers ( host * fleet . Host ) error {
sql := ` SELECT id, username, groupname, uid, user_type FROM host_users WHERE host_id = ? and removed_at IS NULL `
if err := d . db . Select ( & host . Users , sql , host . ID ) ; err != nil {
return errors . Wrap ( err , "load pack stats" )
}
return nil
}
2017-01-04 18:18:21 +00:00
func ( d * Datastore ) DeleteHost ( hid uint ) error {
2020-10-22 17:51:26 +00:00
err := d . deleteEntity ( "hosts" , hid )
2017-01-20 17:22:33 +00:00
if err != nil {
return errors . Wrapf ( err , "deleting host with id %d" , hid )
}
return nil
2016-11-16 13:47:49 +00:00
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) Host ( id uint ) ( * fleet . Host , error ) {
2016-11-16 13:47:49 +00:00
sqlStatement := `
2021-05-31 17:56:50 +00:00
SELECT h . * , t . name AS team_name , ( SELECT additional FROM host_additional WHERE host_id = h . id ) AS additional
2021-05-18 00:52:59 +00:00
FROM hosts h LEFT JOIN teams t ON ( h . team_id = t . id )
2021-05-26 23:24:12 +00:00
WHERE h . id = ?
LIMIT 1
2016-11-16 13:47:49 +00:00
`
2021-06-06 22:07:29 +00:00
host := & fleet . Host { }
2016-11-16 13:47:49 +00:00
err := d . db . Get ( host , sqlStatement , id )
if err != nil {
2021-05-07 04:05:09 +00:00
return nil , errors . Wrap ( err , "get host by id" )
}
if err := d . loadHostPackStats ( host ) ; err != nil {
return nil , err
2016-11-16 13:47:49 +00:00
}
2021-07-13 20:15:38 +00:00
if err := d . loadHostUsers ( host ) ; err != nil {
return nil , err
}
2016-11-16 13:47:49 +00:00
return host , nil
}
2021-07-20 21:39:50 +00:00
func ( d * Datastore ) amountEnrolledHosts ( ) ( int , error ) {
var amount int
err := d . db . Get ( & amount , ` SELECT count(*) FROM hosts ` )
if err != nil {
return 0 , err
}
return amount , nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) ListHosts ( filter fleet . TeamFilter , opt fleet . HostListOptions ) ( [ ] * fleet . Host , error ) {
2021-05-18 00:52:59 +00:00
sql := ` SELECT
2021-07-02 15:59:42 +00:00
h . * ,
2021-05-31 17:56:50 +00:00
t . name AS team_name
Add host additional info filters (#28)
This change adds the ability to filter additional host info via the list hosts endpoint; a continuation from [here](https://github.com/kolide/fleet/pull/2330), but now filtering is accomplished via SQL.
Additional object without filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"time": [
{
"day": "13",
"hour": "3",
"year": "2020",
"month": "10",
"minutes": "43",
"seconds": "11",
"weekday": "Tuesday",
"datetime": "2020-10-13T03:43:11Z",
"iso_8601": "2020-10-13T03:43:11Z",
"timezone": "GMT",
"timestamp": "Tue Oct 13 03:43:11 2020 UTC",
"unix_time": "1602560591",
"local_time": "1602560591",
"local_timezone": "UTC"
}
},
...
```
Additional object with filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts?additional_info_filters=macs,notreal'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"notreal": null
},
...
```
2020-11-14 00:33:25 +00:00
`
2020-03-30 02:19:54 +00:00
var params [ ] interface { }
Add host additional info filters (#28)
This change adds the ability to filter additional host info via the list hosts endpoint; a continuation from [here](https://github.com/kolide/fleet/pull/2330), but now filtering is accomplished via SQL.
Additional object without filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"time": [
{
"day": "13",
"hour": "3",
"year": "2020",
"month": "10",
"minutes": "43",
"seconds": "11",
"weekday": "Tuesday",
"datetime": "2020-10-13T03:43:11Z",
"iso_8601": "2020-10-13T03:43:11Z",
"timezone": "GMT",
"timestamp": "Tue Oct 13 03:43:11 2020 UTC",
"unix_time": "1602560591",
"local_time": "1602560591",
"local_timezone": "UTC"
}
},
...
```
Additional object with filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts?additional_info_filters=macs,notreal'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"notreal": null
},
...
```
2020-11-14 00:33:25 +00:00
2021-05-26 23:24:12 +00:00
// Only include "additional" if filter provided.
if len ( opt . AdditionalFilters ) == 1 && opt . AdditionalFilters [ 0 ] == "*" {
// All info requested.
sql += `
, ( SELECT additional FROM host_additional WHERE host_id = h . id ) AS additional
`
} else if len ( opt . AdditionalFilters ) > 0 {
// Filter specific columns.
sql += ` , ( SELECT JSON_OBJECT (
Add host additional info filters (#28)
This change adds the ability to filter additional host info via the list hosts endpoint; a continuation from [here](https://github.com/kolide/fleet/pull/2330), but now filtering is accomplished via SQL.
Additional object without filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"time": [
{
"day": "13",
"hour": "3",
"year": "2020",
"month": "10",
"minutes": "43",
"seconds": "11",
"weekday": "Tuesday",
"datetime": "2020-10-13T03:43:11Z",
"iso_8601": "2020-10-13T03:43:11Z",
"timezone": "GMT",
"timestamp": "Tue Oct 13 03:43:11 2020 UTC",
"unix_time": "1602560591",
"local_time": "1602560591",
"local_timezone": "UTC"
}
},
...
```
Additional object with filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts?additional_info_filters=macs,notreal'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"notreal": null
},
...
```
2020-11-14 00:33:25 +00:00
`
for _ , field := range opt . AdditionalFilters {
2021-05-17 17:29:50 +00:00
sql += ` ?, JSON_EXTRACT(additional, ?), `
Add host additional info filters (#28)
This change adds the ability to filter additional host info via the list hosts endpoint; a continuation from [here](https://github.com/kolide/fleet/pull/2330), but now filtering is accomplished via SQL.
Additional object without filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"time": [
{
"day": "13",
"hour": "3",
"year": "2020",
"month": "10",
"minutes": "43",
"seconds": "11",
"weekday": "Tuesday",
"datetime": "2020-10-13T03:43:11Z",
"iso_8601": "2020-10-13T03:43:11Z",
"timezone": "GMT",
"timestamp": "Tue Oct 13 03:43:11 2020 UTC",
"unix_time": "1602560591",
"local_time": "1602560591",
"local_timezone": "UTC"
}
},
...
```
Additional object with filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts?additional_info_filters=macs,notreal'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"notreal": null
},
...
```
2020-11-14 00:33:25 +00:00
params = append ( params , field , fmt . Sprintf ( ` $."%s" ` , field ) )
}
sql = sql [ : len ( sql ) - 2 ]
sql += `
2021-05-26 23:24:12 +00:00
) FROM host_additional WHERE host_id = h . id ) AS additional
Add host additional info filters (#28)
This change adds the ability to filter additional host info via the list hosts endpoint; a continuation from [here](https://github.com/kolide/fleet/pull/2330), but now filtering is accomplished via SQL.
Additional object without filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"time": [
{
"day": "13",
"hour": "3",
"year": "2020",
"month": "10",
"minutes": "43",
"seconds": "11",
"weekday": "Tuesday",
"datetime": "2020-10-13T03:43:11Z",
"iso_8601": "2020-10-13T03:43:11Z",
"timezone": "GMT",
"timestamp": "Tue Oct 13 03:43:11 2020 UTC",
"unix_time": "1602560591",
"local_time": "1602560591",
"local_timezone": "UTC"
}
},
...
```
Additional object with filter:
```
curl 'https://localhost:8080/api/v1/kolide/hosts?additional_info_filters=macs,notreal'
...
"additional": {
"macs": [
{
"mac": "00:00:00:00:00:00"
},
{
"mac": "02:42:c0:a8:10:05"
}
],
"notreal": null
},
...
```
2020-11-14 00:33:25 +00:00
`
}
2021-06-04 01:53:43 +00:00
sql += fmt . Sprintf ( ` FROM hosts h LEFT JOIN teams t ON ( h . team_id = t . id )
WHERE TRUE AND % s
` , d . whereFilterHostsByTeams ( filter , "h" ) ,
)
2020-03-30 02:19:54 +00:00
switch opt . StatusFilter {
case "new" :
2021-05-18 00:52:59 +00:00
sql += "AND DATE_ADD(h.created_at, INTERVAL 1 DAY) >= ?"
2020-03-30 02:19:54 +00:00
params = append ( params , time . Now ( ) )
case "online" :
2021-06-06 22:07:29 +00:00
sql += fmt . Sprintf ( "AND DATE_ADD(h.seen_time, INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) > ?" , fleet . OnlineIntervalBuffer )
2020-03-30 02:19:54 +00:00
params = append ( params , time . Now ( ) )
case "offline" :
2021-06-06 22:07:29 +00:00
sql += fmt . Sprintf ( "AND DATE_ADD(h.seen_time, INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(h.seen_time, INTERVAL 30 DAY) >= ?" , fleet . OnlineIntervalBuffer )
2020-03-30 02:19:54 +00:00
params = append ( params , time . Now ( ) , time . Now ( ) )
case "mia" :
2021-05-18 00:52:59 +00:00
sql += "AND DATE_ADD(h.seen_time, INTERVAL 30 DAY) <= ?"
2020-03-30 02:19:54 +00:00
params = append ( params , time . Now ( ) )
}
2021-02-17 00:53:42 +00:00
2021-02-18 20:52:43 +00:00
sql , params = searchLike ( sql , params , opt . MatchQuery , hostSearchColumns ... )
2021-02-17 00:53:42 +00:00
2020-03-30 02:19:54 +00:00
sql = appendListOptionsToSQL ( sql , opt . ListOptions )
2021-06-06 22:07:29 +00:00
hosts := [ ] * fleet . Host { }
2020-03-30 02:19:54 +00:00
if err := d . db . Select ( & hosts , sql , params ... ) ; err != nil {
2017-01-04 21:16:17 +00:00
return nil , errors . Wrap ( err , "list hosts" )
2016-11-16 13:47:49 +00:00
}
2016-12-06 19:51:11 +00:00
return hosts , nil
}
2019-04-09 18:11:11 +00:00
func ( d * Datastore ) CleanupIncomingHosts ( now time . Time ) error {
sqlStatement := `
DELETE FROM hosts
2021-06-24 00:32:19 +00:00
WHERE hostname = ' ' AND osquery_version = ' '
2019-04-09 18:11:11 +00:00
AND created_at < ( ? - INTERVAL 5 MINUTE )
`
if _ , err := d . db . Exec ( sqlStatement , now ) ; err != nil {
return errors . Wrap ( err , "cleanup incoming hosts" )
}
return nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) GenerateHostStatusStatistics ( filter fleet . TeamFilter , now time . Time ) ( online , offline , mia , new uint , e error ) {
2017-04-18 17:39:50 +00:00
// The logic in this function should remain synchronized with
// host.Status and CountHostsInTargets
sqlStatement := fmt . Sprintf ( `
2021-06-04 01:53:43 +00:00
SELECT
COALESCE ( SUM ( CASE WHEN DATE_ADD ( seen_time , INTERVAL 30 DAY ) <= ? THEN 1 ELSE 0 END ) , 0 ) mia ,
COALESCE ( SUM ( CASE WHEN DATE_ADD ( seen_time , INTERVAL LEAST ( distributed_interval , config_tls_refresh ) + % d SECOND ) <= ? AND DATE_ADD ( seen_time , INTERVAL 30 DAY ) >= ? THEN 1 ELSE 0 END ) , 0 ) offline ,
COALESCE ( SUM ( CASE WHEN DATE_ADD ( seen_time , INTERVAL LEAST ( distributed_interval , config_tls_refresh ) + % d SECOND ) > ? THEN 1 ELSE 0 END ) , 0 ) online ,
COALESCE ( SUM ( CASE WHEN DATE_ADD ( created_at , INTERVAL 1 DAY ) >= ? THEN 1 ELSE 0 END ) , 0 ) new
FROM hosts WHERE % s
LIMIT 1 ;
2021-06-06 22:07:29 +00:00
` , fleet . OnlineIntervalBuffer , fleet . OnlineIntervalBuffer ,
2021-06-04 01:53:43 +00:00
d . whereFilterHostsByTeams ( filter , "hosts" ) ,
)
2017-01-04 21:16:17 +00:00
counts := struct {
MIA uint ` db:"mia" `
Offline uint ` db:"offline" `
Online uint ` db:"online" `
2017-01-20 13:57:47 +00:00
New uint ` db:"new" `
2017-01-04 21:16:17 +00:00
} { }
2017-04-18 17:39:50 +00:00
err := d . db . Get ( & counts , sqlStatement , now , now , now , now , now )
2017-01-16 19:52:03 +00:00
if err != nil && err != sql . ErrNoRows {
2017-01-04 21:16:17 +00:00
e = errors . Wrap ( err , "generating host statistics" )
return
}
mia = counts . MIA
offline = counts . Offline
online = counts . Online
2017-01-20 13:57:47 +00:00
new = counts . New
return online , offline , mia , new , nil
2017-01-04 21:16:17 +00:00
}
2016-11-16 13:47:49 +00:00
// EnrollHost enrolls a host
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) EnrollHost ( osqueryHostID , nodeKey string , teamID * uint , cooldown time . Duration ) ( * fleet . Host , error ) {
2016-12-06 19:51:11 +00:00
if osqueryHostID == "" {
2017-01-04 21:16:17 +00:00
return nil , fmt . Errorf ( "missing osquery host identifier" )
2016-11-16 13:47:49 +00:00
}
2016-12-06 19:51:11 +00:00
2021-06-06 22:07:29 +00:00
var host fleet . Host
2020-12-10 19:04:58 +00:00
err := d . withRetryTxx ( func ( tx * sqlx . Tx ) error {
zeroTime := time . Unix ( 0 , 0 ) . Add ( 24 * time . Hour )
2016-11-16 13:47:49 +00:00
2020-12-10 19:04:58 +00:00
var id int64
2021-06-24 00:32:19 +00:00
err := tx . Get ( & host , ` SELECT id, last_enrolled_at FROM hosts WHERE osquery_host_id = ? ` , osqueryHostID )
2021-01-19 22:45:58 +00:00
switch {
case err != nil && ! errors . Is ( err , sql . ErrNoRows ) :
return errors . Wrap ( err , "check existing" )
case errors . Is ( err , sql . ErrNoRows ) :
2020-12-10 19:04:58 +00:00
// Create new host record
sqlInsert := `
INSERT INTO hosts (
2021-06-24 00:32:19 +00:00
detail_updated_at ,
label_updated_at ,
2020-12-10 19:04:58 +00:00
osquery_host_id ,
seen_time ,
node_key ,
2021-05-31 16:02:05 +00:00
team_id
2020-12-10 19:04:58 +00:00
) VALUES ( ? , ? , ? , ? , ? , ? )
`
2021-05-31 16:02:05 +00:00
result , err := tx . Exec ( sqlInsert , zeroTime , zeroTime , osqueryHostID , time . Now ( ) . UTC ( ) , nodeKey , teamID )
2016-11-16 13:47:49 +00:00
2020-12-10 19:04:58 +00:00
if err != nil {
return errors . Wrap ( err , "insert host" )
}
2016-11-16 13:47:49 +00:00
2020-12-10 19:04:58 +00:00
id , _ = result . LastInsertId ( )
2021-01-19 22:45:58 +00:00
default :
2020-12-10 19:04:58 +00:00
// Prevent hosts from enrolling too often with the same identifier.
// Prior to adding this we saw many hosts (probably VMs) with the
// same identifier competing for enrollment and causing perf issues.
2021-06-24 00:32:19 +00:00
if cooldown > 0 && time . Since ( host . LastEnrolledAt ) < cooldown {
2020-12-10 19:04:58 +00:00
return backoff . Permanent ( fmt . Errorf ( "host identified by %s enrolling too often" , osqueryHostID ) )
}
id = int64 ( host . ID )
// Update existing host record
sqlUpdate := `
UPDATE hosts
SET node_key = ? ,
2021-05-31 16:02:05 +00:00
team_id = ? ,
2021-06-24 00:32:19 +00:00
last_enrolled_at = NOW ( )
2020-12-10 19:04:58 +00:00
WHERE osquery_host_id = ?
`
2021-05-31 16:02:05 +00:00
_ , err := tx . Exec ( sqlUpdate , nodeKey , teamID , osqueryHostID )
2016-11-16 13:47:49 +00:00
2020-12-10 19:04:58 +00:00
if err != nil {
return errors . Wrap ( err , "update host" )
}
}
2020-04-07 01:10:20 +00:00
2020-12-10 19:04:58 +00:00
sqlSelect := `
SELECT * FROM hosts WHERE id = ? LIMIT 1
`
err = tx . Get ( & host , sqlSelect , id )
if err != nil {
return errors . Wrap ( err , "getting the host to return" )
}
_ , err = tx . Exec ( ` INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1)) ` , id )
if err != nil {
return errors . Wrap ( err , "insert new host into all hosts label" )
}
2016-11-16 13:47:49 +00:00
2020-12-10 19:04:58 +00:00
return nil
} )
if err != nil {
return nil , err
}
return & host , nil
2016-11-16 13:47:49 +00:00
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) AuthenticateHost ( nodeKey string ) ( * fleet . Host , error ) {
2021-03-09 17:01:26 +00:00
// Select everything besides `additional`
2016-11-16 13:47:49 +00:00
sqlStatement := `
2020-05-21 15:36:00 +00:00
SELECT
id ,
osquery_host_id ,
created_at ,
updated_at ,
2021-06-24 00:32:19 +00:00
detail_updated_at ,
label_updated_at ,
2020-05-21 15:36:00 +00:00
node_key ,
2021-06-24 00:32:19 +00:00
hostname ,
2020-05-21 15:36:00 +00:00
uuid ,
platform ,
osquery_version ,
os_version ,
build ,
platform_like ,
code_name ,
uptime ,
2021-06-24 00:32:19 +00:00
memory ,
2020-05-21 15:36:00 +00:00
cpu_type ,
cpu_subtype ,
cpu_brand ,
cpu_physical_cores ,
cpu_logical_cores ,
hardware_vendor ,
hardware_model ,
hardware_version ,
hardware_serial ,
computer_name ,
primary_ip_id ,
seen_time ,
distributed_interval ,
logger_tls_period ,
2020-05-29 16:12:39 +00:00
config_tls_refresh ,
2021-03-09 17:01:26 +00:00
primary_ip ,
primary_mac ,
2021-05-31 16:02:05 +00:00
refetch_requested ,
team_id
2016-12-06 19:51:11 +00:00
FROM hosts
2020-10-22 17:51:26 +00:00
WHERE node_key = ?
2016-12-06 19:51:11 +00:00
LIMIT 1
2016-11-16 13:47:49 +00:00
`
2016-12-06 19:51:11 +00:00
2021-06-06 22:07:29 +00:00
host := & fleet . Host { }
2016-11-16 13:47:49 +00:00
if err := d . db . Get ( host , sqlStatement , nodeKey ) ; err != nil {
switch err {
case sql . ErrNoRows :
2017-05-25 21:10:12 +00:00
return nil , notFound ( "Host" )
2016-11-16 13:47:49 +00:00
default :
2021-05-31 16:02:05 +00:00
return nil , errors . New ( "find host" )
2016-11-16 13:47:49 +00:00
}
}
2016-12-06 19:51:11 +00:00
return host , nil
2016-11-16 13:47:49 +00:00
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) MarkHostSeen ( host * fleet . Host , t time . Time ) error {
2016-11-16 23:12:59 +00:00
sqlStatement := `
UPDATE hosts SET
2017-01-04 21:16:17 +00:00
seen_time = ?
2016-11-16 23:12:59 +00:00
WHERE node_key = ?
`
_ , err := d . db . Exec ( sqlStatement , t , host . NodeKey )
if err != nil {
2017-01-04 21:16:17 +00:00
return errors . Wrap ( err , "marking host seen" )
2016-11-16 23:12:59 +00:00
}
host . UpdatedAt = t
return nil
2016-11-16 13:47:49 +00:00
}
2021-04-12 23:22:22 +00:00
func ( d * Datastore ) MarkHostsSeen ( hostIDs [ ] uint , t time . Time ) error {
if len ( hostIDs ) == 0 {
return nil
}
if err := d . withRetryTxx ( func ( tx * sqlx . Tx ) error {
query := `
UPDATE hosts SET
seen_time = ?
WHERE id IN ( ? )
`
query , args , err := sqlx . In ( query , t , hostIDs )
if err != nil {
return errors . Wrap ( err , "sqlx in" )
}
query = d . db . Rebind ( query )
if _ , err := d . db . Exec ( query , args ... ) ; err != nil {
return errors . Wrap ( err , "exec update" )
}
return nil
} ) ; err != nil {
return errors . Wrap ( err , "MarkHostsSeen transaction" )
}
return nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) searchHostsWithOmits ( filter fleet . TeamFilter , query string , omit ... uint ) ( [ ] * fleet . Host , error ) {
2019-10-16 17:12:35 +00:00
hostQuery := transformQuery ( query )
2016-12-05 19:16:23 +00:00
ipQuery := ` " ` + query + ` " `
2016-12-01 17:00:00 +00:00
2021-05-25 04:34:08 +00:00
sql := fmt . Sprintf ( `
SELECT DISTINCT *
FROM hosts
WHERE
(
2021-06-24 00:32:19 +00:00
MATCH ( hostname , uuid ) AGAINST ( ? IN BOOLEAN MODE )
2021-05-25 04:34:08 +00:00
OR MATCH ( primary_ip , primary_mac ) AGAINST ( ? IN BOOLEAN MODE )
)
AND id NOT IN ( ? ) AND % s
LIMIT 10
` , d . whereFilterHostsByTeams ( filter , "hosts" ) ,
)
2016-12-05 19:16:23 +00:00
2021-05-25 04:34:08 +00:00
sql , args , err := sqlx . In ( sql , hostQuery , ipQuery , omit )
2016-12-05 19:16:23 +00:00
if err != nil {
2017-01-04 21:16:17 +00:00
return nil , errors . Wrap ( err , "searching hosts" )
2016-11-16 13:47:49 +00:00
}
2016-12-05 19:16:23 +00:00
sql = d . db . Rebind ( sql )
2016-11-16 13:47:49 +00:00
2021-06-06 22:07:29 +00:00
hosts := [ ] * fleet . Host { }
2016-12-06 19:51:11 +00:00
2016-12-05 19:16:23 +00:00
err = d . db . Select ( & hosts , sql , args ... )
if err != nil {
2017-01-04 21:16:17 +00:00
return nil , errors . Wrap ( err , "searching hosts rebound" )
2016-11-16 13:47:49 +00:00
}
return hosts , nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) searchHostsDefault ( filter fleet . TeamFilter , omit ... uint ) ( [ ] * fleet . Host , error ) {
2021-05-25 04:34:08 +00:00
sql := fmt . Sprintf ( `
SELECT * FROM hosts
WHERE id NOT in ( ? ) AND % s
ORDER BY seen_time DESC
LIMIT 5
` , d . whereFilterHostsByTeams ( filter , "hosts" ) ,
)
2017-01-17 14:51:04 +00:00
var in interface { }
{
// use -1 if there are no values to omit.
2020-10-22 17:51:26 +00:00
// Avoids empty args error for `sqlx.In`
2017-01-17 14:51:04 +00:00
in = omit
if len ( omit ) == 0 {
in = - 1
}
}
2021-06-06 22:07:29 +00:00
var hosts [ ] * fleet . Host
2021-05-25 04:34:08 +00:00
sql , args , err := sqlx . In ( sql , in )
2017-01-17 14:51:04 +00:00
if err != nil {
return nil , errors . Wrap ( err , "searching default hosts" )
}
sql = d . db . Rebind ( sql )
err = d . db . Select ( & hosts , sql , args ... )
if err != nil {
return nil , errors . Wrap ( err , "searching default hosts rebound" )
}
return hosts , nil
}
2019-10-16 17:12:35 +00:00
// SearchHosts find hosts by query containing an IP address, a host name or UUID.
// Optionally pass a list of IDs to omit from the search
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) SearchHosts ( filter fleet . TeamFilter , query string , omit ... uint ) ( [ ] * fleet . Host , error ) {
2019-10-16 17:12:35 +00:00
hostQuery := transformQuery ( query )
if ! queryMinLength ( hostQuery ) {
2021-05-25 04:34:08 +00:00
return d . searchHostsDefault ( filter , omit ... )
2017-01-17 14:51:04 +00:00
}
2016-11-16 13:47:49 +00:00
if len ( omit ) > 0 {
2021-05-25 04:34:08 +00:00
return d . searchHostsWithOmits ( filter , query , omit ... )
2016-11-16 13:47:49 +00:00
}
2016-12-05 19:16:23 +00:00
// Needs quotes to avoid each . marking a word boundary
ipQuery := ` " ` + query + ` " `
2016-12-01 17:00:00 +00:00
2021-05-25 04:34:08 +00:00
sql := fmt . Sprintf ( `
SELECT DISTINCT *
FROM hosts
WHERE
(
2021-06-24 00:32:19 +00:00
MATCH ( hostname , uuid ) AGAINST ( ? IN BOOLEAN MODE )
2021-05-25 04:34:08 +00:00
OR MATCH ( primary_ip , primary_mac ) AGAINST ( ? IN BOOLEAN MODE )
) AND % s
LIMIT 10
` , d . whereFilterHostsByTeams ( filter , "hosts" ) ,
)
2016-11-16 13:47:49 +00:00
2021-06-06 22:07:29 +00:00
hosts := [ ] * fleet . Host { }
2021-05-25 04:34:08 +00:00
if err := d . db . Select ( & hosts , sql , hostQuery , ipQuery ) ; err != nil {
2017-01-04 21:16:17 +00:00
return nil , errors . Wrap ( err , "searching hosts" )
2016-11-16 13:47:49 +00:00
}
2016-12-06 19:51:11 +00:00
2016-11-16 13:47:49 +00:00
return hosts , nil
}
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) HostIDsByName ( filter fleet . TeamFilter , hostnames [ ] string ) ( [ ] uint , error ) {
2020-01-24 05:27:20 +00:00
if len ( hostnames ) == 0 {
return [ ] uint { } , nil
}
2021-06-04 01:53:43 +00:00
sqlStatement := fmt . Sprintf ( `
SELECT id FROM hosts
2021-06-24 00:32:19 +00:00
WHERE hostname IN ( ? ) AND % s
2021-06-04 01:53:43 +00:00
` , d . whereFilterHostsByTeams ( filter , "hosts" ) ,
)
2018-05-17 22:54:34 +00:00
sql , args , err := sqlx . In ( sqlStatement , hostnames )
if err != nil {
return nil , errors . Wrap ( err , "building query to get host IDs" )
}
var hostIDs [ ] uint
if err := d . db . Select ( & hostIDs , sql , args ... ) ; err != nil {
return nil , errors . Wrap ( err , "get host IDs" )
}
return hostIDs , nil
}
2020-04-22 20:54:32 +00:00
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) HostByIdentifier ( identifier string ) ( * fleet . Host , error ) {
2020-04-22 20:54:32 +00:00
sql := `
SELECT * FROM hosts
2021-06-24 00:32:19 +00:00
WHERE ? IN ( hostname , osquery_host_id , node_key , uuid )
2020-04-22 20:54:32 +00:00
LIMIT 1
`
2021-06-06 22:07:29 +00:00
host := & fleet . Host { }
2020-04-22 20:54:32 +00:00
err := d . db . Get ( host , sql , identifier )
if err != nil {
return nil , errors . Wrap ( err , "get host by identifier" )
}
2021-05-07 04:05:09 +00:00
if err := d . loadHostPackStats ( host ) ; err != nil {
return nil , err
}
2020-04-22 20:54:32 +00:00
return host , nil
}
2021-05-17 19:23:21 +00:00
func ( d * Datastore ) AddHostsToTeam ( teamID * uint , hostIDs [ ] uint ) error {
if len ( hostIDs ) == 0 {
return nil
}
sql := `
UPDATE hosts SET team_id = ?
WHERE id IN ( ? )
`
sql , args , err := sqlx . In ( sql , teamID , hostIDs )
if err != nil {
return errors . Wrap ( err , "sqlx.In AddHostsToTeam" )
}
if _ , err := d . db . Exec ( sql , args ... ) ; err != nil {
return errors . Wrap ( err , "exec AddHostsToTeam" )
}
return nil
}
2021-05-31 17:56:50 +00:00
2021-06-06 22:07:29 +00:00
func ( d * Datastore ) SaveHostAdditional ( host * fleet . Host ) error {
2021-05-26 23:24:12 +00:00
sql := `
INSERT INTO host_additional ( host_id , additional )
VALUES ( ? , ? )
ON DUPLICATE KEY UPDATE additional = VALUES ( additional )
`
if _ , err := d . db . Exec ( sql , host . ID , host . Additional ) ; err != nil {
return errors . Wrap ( err , "insert additional" )
}
2021-05-31 17:56:50 +00:00
2021-05-26 23:24:12 +00:00
return nil
}
2021-07-13 20:15:38 +00:00
func ( d * Datastore ) SaveHostUsers ( host * fleet . Host ) error {
if len ( host . Users ) == 0 {
if _ , err := d . db . Exec (
` UPDATE host_users SET removed_at = CURRENT_TIMESTAMP WHERE host_id = ? ` ,
host . ID ,
) ; err != nil {
return errors . Wrap ( err , "mark all users as removed" )
}
return nil
}
currentHost := & fleet . Host { ID : host . ID }
if err := d . loadHostUsers ( currentHost ) ; err != nil {
return err
}
incomingUsers := make ( map [ uint ] bool )
var insertArgs [ ] interface { }
for _ , u := range host . Users {
insertArgs = append ( insertArgs , host . ID , u . Uid , u . Username , u . Type , u . GroupName )
incomingUsers [ u . Uid ] = true
}
var removedArgs [ ] interface { }
for _ , u := range currentHost . Users {
if _ , ok := incomingUsers [ u . Uid ] ; ! ok {
removedArgs = append ( removedArgs , u . ID )
}
}
insertValues := strings . TrimSuffix ( strings . Repeat ( "(?, ?, ?, ?, ?)," , len ( host . Users ) ) , "," )
insertSql := fmt . Sprintf (
` INSERT IGNORE INTO host_users (host_id, uid, username, user_type, groupname) VALUES %s ` ,
insertValues ,
)
if _ , err := d . db . Exec ( insertSql , insertArgs ... ) ; err != nil {
return errors . Wrap ( err , "insert users" )
}
if len ( removedArgs ) == 0 {
return nil
}
removedValues := strings . TrimSuffix ( strings . Repeat ( "?," , len ( removedArgs ) ) , "," )
removedSql := fmt . Sprintf (
` UPDATE host_users SET removed_at = CURRENT_TIMESTAMP WHERE id IN (%s) ` ,
removedValues ,
)
if _ , err := d . db . Exec ( removedSql , removedArgs ... ) ; err != nil {
return errors . Wrap ( err , "mark users as removed" )
}
return nil
}