fleet/cmd/fleetctl/upgrade_packs.go

package main

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/fleetdm/fleet/v4/server/fleet"
	"github.com/ghodss/yaml"
	"github.com/urfave/cli/v2"
)

// variable used by tests to set a predictable timestamp
var testUpgradePacksTimestamp time.Time

func upgradePacksCommand() *cli.Command {
	var outputFilename string
	return &cli.Command{
		Name:      "upgrade-packs",
		Usage:     `Generate a config file to assist with converting 2017 "Packs" into portable queries that run on a schedule`,
		UsageText: `fleetctl upgrade-packs [options]`,
		Flags: []cli.Flag{
			configFlag(),
			contextFlag(),
			debugFlag(),
			&cli.StringFlag{
				Name:        "o",
				EnvVars:     []string{"OUTPUT_FILENAME"},
				Value:       "",
				Destination: &outputFilename,
				Usage:       "The name of the file to output converted results",
				Required:    true,
			},
		},
		Action: func(c *cli.Context) error {
			client, err := clientFromCLI(c)
			if err != nil {
				return err
			}

			// must be an admin, but reading packs and queries does not require being an admin,
			// so this must be validated separately, before loading the packs.
			user, err := client.Me()
			if err != nil {
				return fmt.Errorf("check user role: %w", err)
			}
			if user.GlobalRole == nil || *user.GlobalRole != fleet.RoleAdmin {
				return errors.New("could not upgrade packs: forbidden: user does not have the admin role")
			}

			// read the server settings so that the packs URL can be printed in the output
			appCfg, err := client.GetAppConfig()
			if err != nil {
				return fmt.Errorf("could not read configuration: %w", err)
			}

			// read the packs and queries
			packsSpecs, err := client.GetPacksSpecs()
			if err != nil {
				return fmt.Errorf("could not list packs specs: %w", err)
			}
			if len(packsSpecs) == 0 {
				log(c, "No 2017 \"Packs\" found.\n")
				return nil
			}

			// must read the DB packs too (not just the specs) in order to get the
			// host targets of the packs, which are not retrieved by GetPacksSpecs
			// (because host targets cannot be set via the apply spec).
			packsDB, err := client.ListPacks()
			if err != nil {
				return fmt.Errorf("could not list packs: %w", err)
			}
			// map the DB packs by ID
			packsByID := make(map[uint]*fleet.Pack, len(packsDB))
			for _, p := range packsDB {
				packsByID[p.ID] = p
			}

			// get global queries (teamID==nil), because 2017 packs reference global queries.
			queries, err := client.GetQueries(nil, nil)
			if err != nil {
				return fmt.Errorf("could not list queries: %w", err)
			}

			// map queries by packs that reference them
			queriesByPack := mapQueriesToPacks(packsSpecs, queries)

			var (
				newSpecs         []*fleet.QuerySpec
				convertedQueries int
			)
			// use a consistent upgrade timestamp for all new queries (used to make name unique)
			upgradeTimestamp := testUpgradePacksTimestamp
			if upgradeTimestamp.IsZero() {
				upgradeTimestamp = time.Now()
			}
			for _, packSpec := range packsSpecs {
				newPackSpecs, convPackQueries := upgradePackToQueriesSpecs(packSpec, packsByID[packSpec.ID], queriesByPack[packSpec], upgradeTimestamp)
				newSpecs = append(newSpecs, newPackSpecs...)
				convertedQueries += convPackQueries
			}

			if err := writeQuerySpecsToFile(outputFilename, newSpecs); err != nil {
				return fmt.Errorf("could not write queries to file: %w", err)
			}

			log(c, fmt.Sprintf(`Converted %d queries from %d 2017 "Packs" into portable queries:
• For any "Packs" targeting teams, duplicate queries were written for each team.
• For any "Packs" targeting labels or individual hosts, a global query was written without scheduling features enabled.

To import these queries to Fleet, you can merge the data in the output file with your existing query configuration and run `+"`fleetctl apply`"+`.

Note that existing 2017 "Packs" have been left intact. To avoid running duplicate queries on your hosts, visit %s/packs/manage and disable all 2017 "Packs" after upgrading. Fleet will continue to support these until the next major version release, when 2017 "Packs" will be automatically converted to queries.
`, convertedQueries, len(packsSpecs), appCfg.ServerSettings.ServerURL))

			return nil
		},
	}
}

func writeQuerySpecsToFile(filename string, specs []*fleet.QuerySpec) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, defaultFileMode)
	if err != nil {
		return err
	}
	defer f.Close()

	for _, spec := range specs {
		qYaml := fleet.QueryObject{
			ObjectMetadata: fleet.ObjectMetadata{
				ApiVersion: fleet.ApiVersion,
				Kind:       fleet.QueryKind,
			},
			Spec: *spec,
		}
		yml, err := yaml.Marshal(qYaml)
		if err != nil {
			return err
		}
		if _, err := fmt.Fprint(f, string(yml)+"---\n"); err != nil {
			return err
		}
	}

	return f.Close()
}

func mapQueriesToPacks(packs []*fleet.PackSpec, queries []fleet.Query) map[*fleet.PackSpec][]*fleet.Query {
	queriesByName := make(map[string]*fleet.Query, len(queries))
	for _, q := range queries {
		q := q // avoid taking address of iteration var
		queriesByName[q.Name] = &q
	}

	queriesByPack := make(map[*fleet.PackSpec][]*fleet.Query, len(packs))
	for _, pack := range packs {
		for _, sq := range pack.Queries {
			if q := queriesByName[sq.QueryName]; q != nil {
				queriesByPack[pack] = append(queriesByPack[pack], queriesByName[sq.QueryName])
			}
		}
	}
	return queriesByPack
}

// upgrades the pack to the new query format, duplicating queries as needed
// (pack queries targeting teams are duplicated for each team, pack queries
// targeting labels or hosts are duplicated as a global query). Returns the
// generated new query specs and the number of pack queries that were
// converted.
func upgradePackToQueriesSpecs(packSpec *fleet.PackSpec, packDB *fleet.Pack, packQueries []*fleet.Query, ts time.Time) ([]*fleet.QuerySpec, int) {
	if len(packQueries) == 0 {
		// if the pack has no query, there's nothing to convert
		return nil, 0
	}

	var targetsHosts bool
	if packDB != nil {
		targetsHosts = len(packDB.Hosts) > 0
	}

	schedByName := make(map[string]*fleet.PackSpecQuery, len(packSpec.Queries))
	for _, sq := range packSpec.Queries {
		sq := sq // avoid taking the address of iteration var
		schedByName[sq.QueryName] = &sq
	}

	var (
		newSpecs         []*fleet.QuerySpec
		convertedQueries int
	)

	for _, pq := range packQueries {
		sched := schedByName[pq.Name]
		if sched == nil {
			continue
		}

		desc := pq.Description
		if desc != "" && !strings.HasSuffix(desc, "\n") {
			desc += "\n"
		}
		desc += fmt.Sprintf("(converted from pack %q, query %q)", packSpec.Name, pq.Name)

		var loggingType string
		if sched.Snapshot != nil && *sched.Snapshot {
			loggingType = "snapshot"
		} else if sched.Removed != nil {
			if *sched.Removed {
				loggingType = "differential"
			} else {
				loggingType = "differential_ignore_removals"
			}
		}

		var (
			schedPlatform string
			schedVersion  string
			converted     bool
		)
		if sched.Platform != nil {
			schedPlatform = *sched.Platform
		}
		if sched.Version != nil {
			schedVersion = *sched.Version
		}

		for _, tm := range packSpec.Targets.Teams {
			converted = true

			// duplicate the query for each targeted team
			newQueryName := fmt.Sprintf("%s - %s - %s - %s", packSpec.Name, pq.Name, tm, ts.Format("Jan _2 15:04:05.000"))
			newSpecs = append(newSpecs, &fleet.QuerySpec{
				Name:               newQueryName,
				Description:        desc,
				Query:              pq.Query,
				TeamName:           tm,
				Interval:           sched.Interval,
				ObserverCanRun:     pq.ObserverCanRun,
				Platform:           schedPlatform,
				MinOsqueryVersion:  schedVersion,
				AutomationsEnabled: !packSpec.Disabled,
				Logging:            loggingType,
			})
		}

		if len(packSpec.Targets.Labels) > 0 || targetsHosts {
			converted = true

			// write a global query without scheduling features
			newQueryName := fmt.Sprintf("%s - %s - %s", packSpec.Name, pq.Name, ts.Format("Jan _2 15:04:05.000"))
			newSpecs = append(newSpecs, &fleet.QuerySpec{
				Name:               newQueryName,
				Description:        desc,
				Query:              pq.Query,
				TeamName:           "",
				Interval:           0,
				ObserverCanRun:     pq.ObserverCanRun,
				Platform:           schedPlatform,
				MinOsqueryVersion:  schedVersion,
				AutomationsEnabled: false,
				Logging:            loggingType,
			})
		}

		if converted {
			convertedQueries++
		}
	}

	return newSpecs, convertedQueries
}