fleet/server/datastore/mysql/hosts.go

package mysql

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/jmoiron/sqlx"
	"github.com/kolide/fleet/server/kolide"
	"github.com/pkg/errors"
)

func (d *Datastore) NewHost(host *kolide.Host) (*kolide.Host, error) {
	sqlStatement := `
	INSERT INTO hosts (
		osquery_host_id,
		detail_update_time,
		node_key,
		host_name,
		uuid,
		platform,
		osquery_version,
		os_version,
		uptime,
		physical_memory,
		seen_time
	)
	VALUES( ?,?,?,?,?,?,?,?,?,?,? )
	`
	result, err := d.db.Exec(sqlStatement, host.OsqueryHostID, host.DetailUpdateTime,
		host.NodeKey, host.HostName, host.UUID, host.Platform, host.OsqueryVersion,
		host.OSVersion, host.Uptime, host.PhysicalMemory, host.SeenTime)
	if err != nil {
		return nil, errors.Wrap(err, "new host")
	}
	id, _ := result.LastInsertId()
	host.ID = uint(id)
	return host, nil
}

func removedUnusedNics(tx *sqlx.Tx, host *kolide.Host) error {
	if len(host.NetworkInterfaces) == 0 {
		_, err := tx.Exec(`DELETE FROM network_interfaces WHERE host_id = ?`, host.ID)
		return err
	}
	// Remove nics not associated with host
	sqlStatement := fmt.Sprintf(`
			DELETE FROM network_interfaces
			WHERE host_id = %d AND id NOT IN (?)
		`, host.ID)

	list := []uint{}
	for _, nic := range host.NetworkInterfaces {
		list = append(list, nic.ID)
	}

	sql, args, err := sqlx.In(sqlStatement, list)
	if err != nil {
		return err
	}

	sql = tx.Rebind(sql)
	_, err = tx.Exec(sql, args...)
	return err
}

func updateNicsForHost(tx *sqlx.Tx, host *kolide.Host) ([]*kolide.NetworkInterface, error) {
	updatedNics := []*kolide.NetworkInterface{}
	// id = LAST_INSERT_ID(id) is a fix for the lastinsertid not being set
	// properly. See comments in https://goo.gl/cwWRXd.
	sqlStatement := `
	 	INSERT INTO network_interfaces (
			host_id,
			mac,
			ip_address,
			broadcast,
			ibytes,
			interface,
			ipackets,
			last_change,
			mask,
			metric,
			mtu,
			obytes,
			ierrors,
			oerrors,
			opackets,
			point_to_point,
			type
		) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
		ON DUPLICATE KEY UPDATE
			id = LAST_INSERT_ID(id),
			mac = VALUES(mac),
			broadcast = VALUES(broadcast),
			ibytes = VALUES(ibytes),
			ipackets = VALUES(ipackets),
			last_change = VALUES(last_change),
			mask = VALUES(mask),
			metric = VALUES(metric),
			mtu = VALUES(mtu),
			obytes = VALUES(obytes),
			ierrors = VALUES(ierrors),
			oerrors = VALUES(oerrors),
			opackets = VALUES(opackets),
			point_to_point = VALUES(point_to_point),
			type = VALUES(type)
	 `
	for _, nic := range host.NetworkInterfaces {
		nic.HostID = host.ID
		result, err := tx.Exec(sqlStatement,
			nic.HostID,
			nic.MAC,
			nic.IPAddress,
			nic.Broadcast,
			nic.IBytes,
			nic.Interface,
			nic.IPackets,
			nic.LastChange,
			nic.Mask,
			nic.Metric,
			nic.MTU,
			nic.OBytes,
			nic.IErrors,
			nic.OErrors,
			nic.OPackets,
			nic.PointToPoint,
			nic.Type,
		)

		if err != nil {
			return nil, err
		}
		nicID, _ := result.LastInsertId()
		// if row was updated there is no LastInsertID
		if nicID != 0 {
			nic.ID = uint(nicID)
		}
		updatedNics = append(updatedNics, nic)
	}

	return updatedNics, nil
}

// TODO needs test
func (d *Datastore) SaveHost(host *kolide.Host) error {
	sqlStatement := `
		UPDATE hosts SET
			detail_update_time = ?,
			node_key = ?,
			host_name = ?,
			uuid = ?,
			platform = ?,
			osquery_version = ?,
			os_version = ?,
			uptime = ?,
			physical_memory = ?,
			cpu_type = ?,
			cpu_subtype = ?,
			cpu_brand = ?,
			cpu_physical_cores = ?,
			hardware_vendor = ?,
			hardware_model = ?,
			hardware_version = ?,
			hardware_serial = ?,
			computer_name = ?,
			primary_ip_id = ?,
			build = ?,
			platform_like = ?,
			code_name = ?,
			cpu_logical_cores = ?,
			seen_time = ?,
			distributed_interval = ?,
			config_tls_refresh = ?,
			logger_tls_period = ?
		WHERE id = ?
	`

	tx, err := d.db.Beginx()
	if err != nil {
		return errors.Wrap(err, "creating transaction")
	}

	results, err := tx.Exec(sqlStatement,
		host.DetailUpdateTime,
		host.NodeKey,
		host.HostName,
		host.UUID,
		host.Platform,
		host.OsqueryVersion,
		host.OSVersion,
		host.Uptime,
		host.PhysicalMemory,
		host.CPUType,
		host.CPUSubtype,
		host.CPUBrand,
		host.CPUPhysicalCores,
		host.HardwareVendor,
		host.HardwareModel,
		host.HardwareVersion,
		host.HardwareSerial,
		host.ComputerName,
		host.PrimaryNetworkInterfaceID,
		host.Build,
		host.PlatformLike,
		host.CodeName,
		host.CPULogicalCores,
		host.SeenTime,
		host.DistributedInterval,
		host.ConfigTLSRefresh,
		host.LoggerTLSPeriod,
		host.ID)
	if err != nil {
		tx.Rollback()
		return errors.Wrap(err, "executing main SQL statement")
	}
	rowsAffected, err := results.RowsAffected()
	if err != nil {
		tx.Rollback()
		return errors.Wrap(err, "rows affected updating host")
	}
	if rowsAffected == 0 {
		tx.Rollback()
		return notFound("Host").WithID(host.ID)
	}

	host.NetworkInterfaces, err = updateNicsForHost(tx, host)
	if err != nil {
		tx.Rollback()
		return errors.Wrap(err, "updating nics")
	}

	if err = removedUnusedNics(tx, host); err != nil {
		tx.Rollback()
		return errors.Wrap(err, "removing unused nics")
	}

	if needsUpdate := host.ResetPrimaryNetwork(); needsUpdate {
		results, err = tx.Exec(
			"UPDATE hosts SET primary_ip_id = ? WHERE id = ?",
			host.PrimaryNetworkInterfaceID,
			host.ID,
		)

		if err != nil {
			tx.Rollback()
			return errors.Wrap(err, "resetting primary network")
		}
		rowsAffected, err = results.RowsAffected()
		if err != nil {
			tx.Rollback()
			return errors.Wrap(err, "rows affected resetting primary network")
		}
		if rowsAffected == 0 {
			tx.Rollback()
			return notFound("Host").WithID(host.ID)
		}
	}

	if err = tx.Commit(); err != nil {
		tx.Rollback()
		return errors.Wrap(err, "committing transaction")
	}
	return nil
}

func (d *Datastore) DeleteHost(hid uint) error {
	_, err := d.db.Exec("DELETE FROM hosts WHERE id = ?", hid)
	if err != nil {
		return errors.Wrapf(err, "deleting host with id %d", hid)
	}
	return nil
}

// TODO needs test
func (d *Datastore) Host(id uint) (*kolide.Host, error) {
	sqlStatement := `
		SELECT * FROM hosts
		WHERE id = ? AND NOT deleted LIMIT 1
	`
	host := &kolide.Host{}
	err := d.db.Get(host, sqlStatement, id)
	if err != nil {
		return nil, errors.Wrap(err, "getting host by id")
	}

	if err := d.getNetInterfacesForHost(host); err != nil {
		return nil, err
	}

	return host, nil

}

func (d *Datastore) ListHosts(opt kolide.ListOptions) ([]*kolide.Host, error) {
	sqlStatement := `
		SELECT * FROM hosts
		WHERE NOT deleted
	`
	sqlStatement = appendListOptionsToSQL(sqlStatement, opt)
	hosts := []*kolide.Host{}
	if err := d.db.Select(&hosts, sqlStatement); err != nil {
		return nil, errors.Wrap(err, "list hosts")
	}

	if opt.PerPage == 0 || (opt.Page == 0 && uint(len(hosts)) < opt.PerPage) {
		// If all hosts, we can use the optimized network interface retrieval function
		if err := d.getNetInterfacesForAllHosts(hosts); err != nil {
			return nil, err
		}

	} else {
		if err := d.getNetInterfacesForHosts(hosts); err != nil {
			return nil, err
		}
	}

	return hosts, nil
}

func (d *Datastore) GenerateHostStatusStatistics(now time.Time) (online, offline, mia, new uint, e error) {
	// The logic in this function should remain synchronized with
	// host.Status and CountHostsInTargets

	sqlStatement := fmt.Sprintf(`
		SELECT
			COALESCE(SUM(CASE WHEN DATE_ADD(seen_time, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia,
			COALESCE(SUM(CASE WHEN DATE_ADD(seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? AND DATE_ADD(seen_time, INTERVAL 30 DAY) >= ? THEN 1 ELSE 0 END), 0) offline,
			COALESCE(SUM(CASE WHEN DATE_ADD(seen_time, INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online,
			COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new
		FROM hosts
		LIMIT 1;
	`, kolide.OnlineIntervalBuffer, kolide.OnlineIntervalBuffer)

	counts := struct {
		MIA     uint `db:"mia"`
		Offline uint `db:"offline"`
		Online  uint `db:"online"`
		New     uint `db:"new"`
	}{}
	err := d.db.Get(&counts, sqlStatement, now, now, now, now, now)
	if err != nil && err != sql.ErrNoRows {
		e = errors.Wrap(err, "generating host statistics")
		return
	}

	mia = counts.MIA
	offline = counts.Offline
	online = counts.Online
	new = counts.New
	return online, offline, mia, new, nil
}

// Optimized network interface fetch for sets of hosts.  Instead of looping
// through hosts and doing a select for each host to get nics, we get all
// nics at once, so 2 db calls, and then assign nics to hosts here.
func (d *Datastore) getNetInterfacesForHosts(hosts []*kolide.Host) error {
	if len(hosts) == 0 {
		return nil
	}

	sqlStatement := `
		SELECT *
		FROM network_interfaces
		WHERE host_id IN (:hosts)
		ORDER BY host_id ASC
	`
	hostIDs := make([]uint, len(hosts))

	for _, host := range hosts {
		hostIDs = append(hostIDs, host.ID)
	}

	arg := map[string]interface{}{
		"hosts": hostIDs,
	}
	query, args, err := sqlx.Named(sqlStatement, arg)
	if err != nil {
		return errors.Wrap(err, "select nics for hosts, named query")
	}

	query, args, err = sqlx.In(query, args...)
	if err != nil {
		return errors.Wrap(err, "select nics for hosts, in query")
	}

	query = d.db.Rebind(query)
	nics := []*kolide.NetworkInterface{}
	err = d.db.Select(&nics, query, args...)
	if err != nil {
		return errors.Wrap(err, "select nics for hosts, rebound query")
	}

	for _, host := range hosts {
		for i := 0; i < len(nics); i++ {
			if host.ID == nics[i].HostID {
				host.NetworkInterfaces = append(host.NetworkInterfaces, nics[i])
			}
		}
	}

	return nil
}

// When we know we're loading the network interfaces for all hosts, we can skip
// the IN clause and load them all. This allows us to load net interfaces
// without error for larger sets of hosts.
func (d *Datastore) getNetInterfacesForAllHosts(hosts []*kolide.Host) error {
	if len(hosts) == 0 {
		return nil
	}

	sqlStatement := `
		SELECT *
		FROM network_interfaces
		ORDER BY host_id ASC
	`
	nics := []*kolide.NetworkInterface{}
	err := d.db.Select(&nics, sqlStatement)
	if err != nil {
		return errors.Wrap(err, "select nics for all hosts")
	}

	for _, host := range hosts {
		for i := 0; i < len(nics); i++ {
			if host.ID == nics[i].HostID {
				host.NetworkInterfaces = append(host.NetworkInterfaces, nics[i])
			}
		}
	}

	return nil
}

func (d *Datastore) getNetInterfacesForHost(host *kolide.Host) error {
	sqlStatement := `
		SELECT * FROM network_interfaces
		WHERE host_id = ?
	`
	return d.db.Select(&host.NetworkInterfaces, sqlStatement, host.ID)
}

// EnrollHost enrolls a host
func (d *Datastore) EnrollHost(osqueryHostID string, nodeKeySize int) (*kolide.Host, error) {
	if osqueryHostID == "" {
		return nil, fmt.Errorf("missing osquery host identifier")
	}

	detailUpdateTime := time.Unix(0, 0).Add(24 * time.Hour)
	nodeKey, err := kolide.RandomText(nodeKeySize)
	if err != nil {
		return nil, errors.Wrap(err, "generating random text")
	}

	sqlInsert := `
		INSERT INTO hosts (
			detail_update_time,
			osquery_host_id,
			seen_time,
			node_key
		) VALUES (?, ?, ?, ?)
		ON DUPLICATE KEY UPDATE
			node_key = VALUES(node_key),
			deleted = FALSE
	`

	var result sql.Result

	result, err = d.db.Exec(sqlInsert, detailUpdateTime, osqueryHostID, time.Now().UTC(), nodeKey)

	if err != nil {
		return nil, errors.Wrap(err, "inserting")
	}

	id, _ := result.LastInsertId()
	sqlSelect := `
		SELECT * FROM hosts WHERE id = ? LIMIT 1
	`
	host := &kolide.Host{}
	err = d.db.Get(host, sqlSelect, id)
	if err != nil {
		return nil, errors.Wrap(err, "getting the host to return")
	}

	return host, nil

}

func (d *Datastore) AuthenticateHost(nodeKey string) (*kolide.Host, error) {
	sqlStatement := `
		SELECT *
		FROM hosts
		WHERE node_key = ? AND NOT deleted
		LIMIT 1
	`

	host := &kolide.Host{}
	if err := d.db.Get(host, sqlStatement, nodeKey); err != nil {
		switch err {
		case sql.ErrNoRows:
			return nil, notFound("Host")
		default:
			return nil, errors.New("finding host")
		}
	}

	if err := d.getNetInterfacesForHost(host); err != nil {
		return nil, errors.Wrap(err, "getting interfaces")
	}

	return host, nil
}

func (d *Datastore) MarkHostSeen(host *kolide.Host, t time.Time) error {
	sqlStatement := `
		UPDATE hosts SET
			seen_time = ?
		WHERE node_key=?
	`

	_, err := d.db.Exec(sqlStatement, t, host.NodeKey)
	if err != nil {
		return errors.Wrap(err, "marking host seen")
	}

	host.UpdatedAt = t
	return nil
}

func (d *Datastore) searchHostsWithOmits(query string, omit ...uint) ([]*kolide.Host, error) {
	hostnameQuery := query
	if len(hostnameQuery) > 0 {
		hostnameQuery += "*"
	}

	ipQuery := `"` + query + `"`

	sqlStatement :=
		`
		SELECT DISTINCT *
		FROM hosts
		WHERE
		(
			id IN (
				SELECT id
				FROM hosts
				WHERE
				MATCH(host_name) AGAINST(? IN BOOLEAN MODE)
			)
		OR
			id IN (
				SELECT host_id
				FROM network_interfaces
				WHERE
				MATCH(ip_address) AGAINST(? IN BOOLEAN MODE)
			)
		)
		AND NOT deleted
		AND id NOT IN (?)
		LIMIT 10
	`

	sql, args, err := sqlx.In(sqlStatement, hostnameQuery, ipQuery, omit)
	if err != nil {
		return nil, errors.Wrap(err, "searching hosts")
	}
	sql = d.db.Rebind(sql)

	hosts := []*kolide.Host{}

	err = d.db.Select(&hosts, sql, args...)
	if err != nil {
		return nil, errors.Wrap(err, "searching hosts rebound")
	}

	if err := d.getNetInterfacesForHosts(hosts); err != nil {
		return nil, errors.Wrap(err, "getting network interfaces for hosts")
	}

	return hosts, nil
}

func (d *Datastore) searchHostsDefault(omit ...uint) ([]*kolide.Host, error) {
	sqlStatement := `
	SELECT * FROM hosts
	WHERE NOT deleted
	AND id NOT IN (?)
	ORDER BY seen_time DESC
	LIMIT 5
	`

	var in interface{}
	{
		// use -1 if there are no values to omit.
		//Avoids empty args error for `sqlx.In`
		in = omit
		if len(omit) == 0 {
			in = -1
		}
	}

	var hosts []*kolide.Host
	sql, args, err := sqlx.In(sqlStatement, in)
	if err != nil {
		return nil, errors.Wrap(err, "searching default hosts")
	}
	sql = d.db.Rebind(sql)
	err = d.db.Select(&hosts, sql, args...)
	if err != nil {
		return nil, errors.Wrap(err, "searching default hosts rebound")
	}
	if err := d.getNetInterfacesForHosts(hosts); err != nil {
		return nil, errors.Wrap(err, "getting network interfaces for default search hosts")
	}
	return hosts, nil
}

// SearchHosts find hosts by query containing an IP address or a host name. Optionally
// pass a list of IDs to omit from the search
func (d *Datastore) SearchHosts(query string, omit ...uint) ([]*kolide.Host, error) {
	if query == "" {
		return d.searchHostsDefault(omit...)
	}
	if len(omit) > 0 {
		return d.searchHostsWithOmits(query, omit...)
	}

	hostnameQuery := query
	hostnameQuery += "*"

	// Needs quotes to avoid each . marking a word boundary
	ipQuery := `"` + query + `"`

	sqlStatement :=
		`
		SELECT DISTINCT *
		FROM hosts
		WHERE
		(
			id IN (
				SELECT id
				FROM hosts
				WHERE
				MATCH(host_name) AGAINST(? IN BOOLEAN MODE)
			)
		OR
			id IN (
				SELECT host_id
				FROM network_interfaces
				WHERE
				MATCH(ip_address) AGAINST(? IN BOOLEAN MODE)
			)
		)
		AND NOT deleted
		LIMIT 10
	`
	hosts := []*kolide.Host{}

	if err := d.db.Select(&hosts, sqlStatement, hostnameQuery, ipQuery); err != nil {
		return nil, errors.Wrap(err, "searching hosts")
	}

	if err := d.getNetInterfacesForHosts(hosts); err != nil {
		return nil, errors.Wrap(err, "getting interfaces")
	}

	return hosts, nil

}

func (d *Datastore) DistributedQueriesForHost(host *kolide.Host) (map[uint]string, error) {
	sqlStatement := `
		SELECT DISTINCT dqc.id, q.query
		FROM distributed_query_campaigns dqc
		JOIN distributed_query_campaign_targets dqct
		    ON (dqc.id = dqct.distributed_query_campaign_id)
		LEFT JOIN label_query_executions lqe
		    ON (dqct.type = ? AND dqct.target_id = lqe.label_id AND lqe.matches)
		LEFT JOIN hosts h
		    ON ((dqct.type = ? AND lqe.host_id = h.id) OR (dqct.type = ? AND dqct.target_id = h.id))
		LEFT JOIN distributed_query_executions dqe
		    ON (h.id = dqe.host_id AND dqc.id = dqe.distributed_query_campaign_id)
		JOIN queries q
		    ON (dqc.query_id = q.id)
		WHERE dqe.status IS NULL AND dqc.status = ? AND h.id = ?
			AND NOT q.deleted
			AND NOT dqc.deleted
 `
	rows, err := d.db.Query(sqlStatement, kolide.TargetLabel, kolide.TargetLabel,
		kolide.TargetHost, kolide.QueryRunning, host.ID)
	if err != nil {
		return nil, errors.Wrap(err, "finding distributed queries for host")
	}
	defer rows.Close()

	results := map[uint]string{}

	for rows.Next() {
		var (
			id    uint
			query string
		)
		err = rows.Scan(&id, &query)
		if err != nil {
			return nil, errors.Wrap(err, "scanning query results")
		}

		results[id] = query

	}

	return results, nil
}

func (d *Datastore) HostIDsByName(hostnames []string) ([]uint, error) {
	sqlStatement := `
		SELECT id FROM hosts
		WHERE host_name IN (?)
	`

	sql, args, err := sqlx.In(sqlStatement, hostnames)
	if err != nil {
		return nil, errors.Wrap(err, "building query to get host IDs")
	}

	var hostIDs []uint
	if err := d.db.Select(&hostIDs, sql, args...); err != nil {
		return nil, errors.Wrap(err, "get host IDs")
	}

	return hostIDs, nil

}