package mysql import ( "database/sql" "fmt" "time" "github.com/cenkalti/backoff/v4" "github.com/fleetdm/fleet/server/kolide" "github.com/jmoiron/sqlx" "github.com/pkg/errors" ) var hostSearchColumns = []string{"host_name", "uuid", "hardware_serial", "primary_ip"} func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) { sqlStatement := ` INSERT INTO hosts ( osquery_host_id, detail_update_time, label_update_time, node_key, host_name, uuid, platform, osquery_version, os_version, uptime, physical_memory, seen_time ) VALUES( ?,?,?,?,?,?,?,?,?,?,?,? ) ` result, err := d.db.Exec( sqlStatement, host.OsqueryHostID, host.DetailUpdateTime, host.LabelUpdateTime, host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion, host.OSVersion, host.Uptime, host.PhysicalMemory, host.SeenTime, ) if err != nil { return nil, errors.Wrap(err, "new host") } id, _ := result.LastInsertId() host.ID = uint(id) return host, nil } // TODO needs test func (d *Datastore) SaveHost(host *kolide.Host) error { sqlStatement := ` UPDATE hosts SET detail_update_time = ?, label_update_time = ?, node_key = ?, host_name = ?, uuid = ?, platform = ?, osquery_version = ?, os_version = ?, uptime = ?, physical_memory = ?, cpu_type = ?, cpu_subtype = ?, cpu_brand = ?, cpu_physical_cores = ?, hardware_vendor = ?, hardware_model = ?, hardware_version = ?, hardware_serial = ?, computer_name = ?, build = ?, platform_like = ?, code_name = ?, cpu_logical_cores = ?, seen_time = ?, distributed_interval = ?, config_tls_refresh = ?, logger_tls_period = ?, additional = COALESCE(?, additional), enroll_secret_name = ?, primary_ip = ?, primary_mac = ? WHERE id = ? ` _, err := d.db.Exec(sqlStatement, host.DetailUpdateTime, host.LabelUpdateTime, host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion, host.OSVersion, host.Uptime, host.PhysicalMemory, host.CPUType, host.CPUSubtype, host.CPUBrand, host.CPUPhysicalCores, host.HardwareVendor, host.HardwareModel, host.HardwareVersion, host.HardwareSerial, host.ComputerName, host.Build, host.PlatformLike, host.CodeName, host.CPULogicalCores, host.SeenTime, host.DistributedInterval, host.ConfigTLSRefresh, host.LoggerTLSPeriod, host.Additional, host.EnrollSecretName, host.PrimaryIP, host.PrimaryMac, host.ID, ) if err != nil { return errors.Wrapf(err, "save host with id %d", host.ID) } return nil } func (d *Datastore) DeleteHost(hid uint) error { err := d.deleteEntity("hosts", hid) if err != nil { return errors.Wrapf(err, "deleting host with id %d", hid) } return nil } func (d *Datastore) Host(id uint) (*kolide.Host, error) { sqlStatement := ` SELECT * FROM hosts WHERE id = ? LIMIT 1 ` host := &kolide.Host{} err := d.db.Get(host, sqlStatement, id) if err != nil { return nil, errors.Wrap(err, "getting host by id") } return host, nil } func (d *Datastore) ListHosts(opt kolide.HostListOptions) ([]*kolide.Host, error) { sql := `SELECT id, osquery_host_id, created_at, updated_at, detail_update_time, node_key, host_name, uuid, platform, osquery_version, os_version, build, platform_like, code_name, uptime, physical_memory, cpu_type, cpu_subtype, cpu_brand, cpu_physical_cores, cpu_logical_cores, hardware_vendor, hardware_model, hardware_version, hardware_serial, computer_name, primary_ip_id, seen_time, distributed_interval, logger_tls_period, config_tls_refresh, primary_ip, primary_mac, label_update_time, enroll_secret_name, ` var params []interface{} // Filter additional info by extracting into a new json object. if len(opt.AdditionalFilters) > 0 { sql += `JSON_OBJECT( ` for _, field := range opt.AdditionalFilters { sql += fmt.Sprintf(`?, JSON_EXTRACT(additional, ?), `) params = append(params, field, fmt.Sprintf(`$."%s"`, field)) } sql = sql[:len(sql)-2] sql += ` ) AS additional ` } else { sql += ` additional ` } sql += `FROM hosts WHERE TRUE ` switch opt.StatusFilter { case "new": sql += "AND DATE_ADD(created_at, INTERVAL 1 DAY) >= ?" params = append(params, time.Now()) case "online": sql += fmt.Sprintf("AND DATE_ADD(seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ?", kolide.OnlineIntervalBuffer) params = append(params, time.Now()) case "offline": sql += fmt.Sprintf("AND DATE_ADD(seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(seen_time, INTERVAL 30 DAY) >= ?", kolide.OnlineIntervalBuffer) params = append(params, time.Now(), time.Now()) case "mia": sql += "AND DATE_ADD(seen_time, INTERVAL 30 DAY) <= ?" params = append(params, time.Now()) } sql, params = searchLike(sql, params, opt.MatchQuery, hostSearchColumns...) sql = appendListOptionsToSQL(sql, opt.ListOptions) hosts := []*kolide.Host{} if err := d.db.Select(&hosts, sql, params...); err != nil { return nil, errors.Wrap(err, "list hosts") } return hosts, nil } func (d *Datastore) CleanupIncomingHosts(now time.Time) error { sqlStatement := ` DELETE FROM hosts WHERE host_name = '' AND osquery_version = '' AND created_at < (? - INTERVAL 5 MINUTE) ` if _, err := d.db.Exec(sqlStatement, now); err != nil { return errors.Wrap(err, "cleanup incoming hosts") } return nil } func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia, new uint, e error) { // The logic in this function should remain synchronized with // host.Status and CountHostsInTargets sqlStatement := fmt.Sprintf(` SELECT COALESCE(SUM(CASE WHEN DATE_ADD(seen_time, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia, COALESCE(SUM(CASE WHEN DATE_ADD(seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(seen_time, INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline, COALESCE(SUM(CASE WHEN DATE_ADD(seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online, COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new FROM hosts LIMIT 1; `, kolide.OnlineIntervalBuffer, kolide.OnlineIntervalBuffer) counts := struct { MIA uint `db:"mia"` Offline uint `db:"offline"` Online uint `db:"online"` New uint `db:"new"` }{} err := d.db.Get(&counts, sqlStatement, now, now, now, now, now) if err != nil && err != sql.ErrNoRows { e = errors.Wrap(err, "generating host statistics") return } mia = counts.MIA offline = counts.Offline online = counts.Online new = counts.New return online, offline, mia, new, nil } // EnrollHost enrolls a host func (d *Datastore) EnrollHost(osqueryHostID, nodeKey, secretName string, cooldown time.Duration) (*kolide.Host, error) { if osqueryHostID == "" { return nil, fmt.Errorf("missing osquery host identifier") } var host kolide.Host err := d.withRetryTxx(func(tx *sqlx.Tx) error { zeroTime := time.Unix(0, 0).Add(24 * time.Hour) var id int64 err := tx.Get(&host, `SELECT id, last_enroll_time FROM hosts WHERE osquery_host_id = ?`, osqueryHostID) switch { case err != nil && !errors.Is(err, sql.ErrNoRows): return errors.Wrap(err, "check existing") case errors.Is(err, sql.ErrNoRows): // Create new host record sqlInsert := ` INSERT INTO hosts ( detail_update_time, label_update_time, osquery_host_id, seen_time, node_key, enroll_secret_name ) VALUES (?, ?, ?, ?, ?, ?) ` result, err := tx.Exec(sqlInsert, zeroTime, zeroTime, osqueryHostID, time.Now().UTC(), nodeKey, secretName) if err != nil { return errors.Wrap(err, "insert host") } id, _ = result.LastInsertId() default: // Prevent hosts from enrolling too often with the same identifier. // Prior to adding this we saw many hosts (probably VMs) with the // same identifier competing for enrollment and causing perf issues. if cooldown > 0 && time.Since(host.LastEnrollTime) < cooldown { return backoff.Permanent(fmt.Errorf("host identified by %s enrolling too often", osqueryHostID)) } id = int64(host.ID) // Update existing host record sqlUpdate := ` UPDATE hosts SET node_key = ?, enroll_secret_name = ?, last_enroll_time = NOW() WHERE osquery_host_id = ? ` _, err := tx.Exec(sqlUpdate, nodeKey, secretName, osqueryHostID) if err != nil { return errors.Wrap(err, "update host") } } sqlSelect := ` SELECT * FROM hosts WHERE id = ? LIMIT 1 ` err = tx.Get(&host, sqlSelect, id) if err != nil { return errors.Wrap(err, "getting the host to return") } _, err = tx.Exec(`INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, (SELECT id FROM labels WHERE name = 'All Hosts' AND label_type = 1))`, id) if err != nil { return errors.Wrap(err, "insert new host into all hosts label") } return nil }) if err != nil { return nil, err } return &host, nil } func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) { // Select everything besides `additional` sqlStatement := ` SELECT id, osquery_host_id, created_at, updated_at, detail_update_time, label_update_time, node_key, host_name, uuid, platform, osquery_version, os_version, build, platform_like, code_name, uptime, physical_memory, cpu_type, cpu_subtype, cpu_brand, cpu_physical_cores, cpu_logical_cores, hardware_vendor, hardware_model, hardware_version, hardware_serial, computer_name, primary_ip_id, seen_time, distributed_interval, logger_tls_period, config_tls_refresh, primary_ip, primary_mac, enroll_secret_name FROM hosts WHERE node_key = ? LIMIT 1 ` host := &kolide.Host{} if err := d.db.Get(host, sqlStatement, nodeKey); err != nil { switch err { case sql.ErrNoRows: return nil, notFound("Host") default: return nil, errors.New("finding host") } } return host, nil } func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error { sqlStatement := ` UPDATE hosts SET seen_time = ? WHERE node_key=? ` _, err := d.db.Exec(sqlStatement, t, host.NodeKey) if err != nil { return errors.Wrap(err, "marking host seen") } host.UpdatedAt = t return nil } func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide.Host, error) { hostQuery := transformQuery(query) ipQuery := `"` + query + `"` sqlStatement := ` SELECT DISTINCT * FROM hosts WHERE ( MATCH (host_name, uuid) AGAINST (? IN BOOLEAN MODE) OR MATCH (primary_ip, primary_mac) AGAINST (? IN BOOLEAN MODE) ) AND id NOT IN (?) LIMIT 10 ` sql, args, err := sqlx.In(sqlStatement, hostQuery, ipQuery, omit) if err != nil { return nil, errors.Wrap(err, "searching hosts") } sql = d.db.Rebind(sql) hosts := []*kolide.Host{} err = d.db.Select(&hosts, sql, args...) if err != nil { return nil, errors.Wrap(err, "searching hosts rebound") } return hosts, nil } func (d *Datastore) searchHostsDefault(omit ...uint) ([]*kolide.Host, error) { sqlStatement := ` SELECT * FROM hosts WHERE id NOT in (?) ORDER BY seen_time DESC LIMIT 5 ` var in interface{} { // use -1 if there are no values to omit. // Avoids empty args error for `sqlx.In` in = omit if len(omit) == 0 { in = -1 } } var hosts []*kolide.Host sql, args, err := sqlx.In(sqlStatement, in) if err != nil { return nil, errors.Wrap(err, "searching default hosts") } sql = d.db.Rebind(sql) err = d.db.Select(&hosts, sql, args...) if err != nil { return nil, errors.Wrap(err, "searching default hosts rebound") } return hosts, nil } // SearchHosts find hosts by query containing an IP address, a host name or UUID. // Optionally pass a list of IDs to omit from the search func (d *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, error) { hostQuery := transformQuery(query) if !queryMinLength(hostQuery) { return d.searchHostsDefault(omit...) } if len(omit) > 0 { return d.searchHostsWithOmits(query, omit...) } // Needs quotes to avoid each . marking a word boundary ipQuery := `"` + query + `"` sqlStatement := ` SELECT DISTINCT * FROM hosts WHERE ( MATCH (host_name, uuid) AGAINST (? IN BOOLEAN MODE) OR MATCH (primary_ip, primary_mac) AGAINST (? IN BOOLEAN MODE) ) LIMIT 10 ` hosts := []*kolide.Host{} if err := d.db.Select(&hosts, sqlStatement, hostQuery, ipQuery); err != nil { return nil, errors.Wrap(err, "searching hosts") } return hosts, nil } func (d *Datastore) HostIDsByName(hostnames []string) ([]uint, error) { if len(hostnames) == 0 { return []uint{}, nil } sqlStatement := ` SELECT id FROM hosts WHERE host_name IN (?) ` sql, args, err := sqlx.In(sqlStatement, hostnames) if err != nil { return nil, errors.Wrap(err, "building query to get host IDs") } var hostIDs []uint if err := d.db.Select(&hostIDs, sql, args...); err != nil { return nil, errors.Wrap(err, "get host IDs") } return hostIDs, nil } func (d *Datastore) HostByIdentifier(identifier string) (*kolide.Host, error) { sql := ` SELECT * FROM hosts WHERE ? IN (host_name, osquery_host_id, node_key, uuid) LIMIT 1 ` host := &kolide.Host{} err := d.db.Get(host, sql, identifier) if err != nil { return nil, errors.Wrap(err, "get host by identifier") } return host, nil }