added package command from orbit as fleetctl command (#1802)

* added package command from orbit as fleetctl command

* update deployment docs

* add changes file

* added tests for package command, run go mod tidy & go mod verify

* validate that package files exist

* comment out msi packaging test until we can investigate github runner permission issues

.gitignore

@@ -49,3 +49,6 @@ backup.sql.gz
 # Common mistake for new developers to run npm install and then end up
 # committing a package-lock.json. Fleet app uses Yarn with yarn.lock.
 package-lock.json
+
+# generated installers
+orbit-osquery*

changes/issue-541-fleetctl-package

@@ -0,0 +1 @@
+* added `fleetctl package` command -- originally from the orbit package/project this simply enables one to run package with fleetctl

cmd/fleetctl/fleetctl.go

@@ -61,6 +61,7 @@ func createApp(reader io.Reader, writer io.Writer, exitErrHandler cli.ExitErrHan
 		previewCommand(),
 		eefleetctl.UpdatesCommand(),
 		hostsCommand(),
+		packageCommand(),
 	}
 	return app
 }

cmd/fleetctl/package.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"github.com/fleetdm/fleet/v4/orbit/pkg/packaging"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+)
+
+var opt packaging.Options
+
+func packageCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "package",
+		Aliases:     nil,
+		Usage:       "Create an Orbit installer package",
+		Description: "An easy way to create fully boot-strapped installer packages for Windows, macOS, or Linux",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "type",
+				Usage:    "Type of package to build",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:        "enroll-secret",
+				Usage:       "Enroll secret for authenticating to Fleet server",
+				Destination: &opt.EnrollSecret,
+			},
+			&cli.StringFlag{
+				Name:        "fleet-url",
+				Usage:       "URL (host:port) of Fleet server",
+				Destination: &opt.FleetURL,
+			},
+			&cli.StringFlag{
+				Name:        "fleet-certificate",
+				Usage:       "Path to server cerificate bundle",
+				Destination: &opt.FleetCertificate,
+			},
+			&cli.StringFlag{
+				Name:        "identifier",
+				Usage:       "Identifier for package product",
+				Value:       "com.fleetdm.orbit",
+				Destination: &opt.Identifier,
+			},
+			&cli.StringFlag{
+				Name:        "version",
+				Usage:       "Version for package product",
+				Value:       "0.0.3",
+				Destination: &opt.Version,
+			},
+			&cli.BoolFlag{
+				Name:        "insecure",
+				Usage:       "Disable TLS certificate verification",
+				Destination: &opt.Insecure,
+			},
+			&cli.BoolFlag{
+				Name:        "service",
+				Usage:       "Install orbit/osquery with a persistence service (launchd, systemd, etc.)",
+				Value:       true,
+				Destination: &opt.StartService,
+			},
+			&cli.StringFlag{
+				Name:        "sign-identity",
+				Usage:       "Identity to use for macOS codesigning",
+				Destination: &opt.SignIdentity,
+			},
+			&cli.BoolFlag{
+				Name:        "notarize",
+				Usage:       "Whether to notarize macOS packages",
+				Destination: &opt.Notarize,
+			},
+			&cli.StringFlag{
+				Name:        "osqueryd-channel",
+				Usage:       "Update channel of osqueryd to use",
+				Value:       "stable",
+				Destination: &opt.OsquerydChannel,
+			},
+			&cli.StringFlag{
+				Name:        "orbit-channel",
+				Usage:       "Update channel of Orbit to use",
+				Value:       "stable",
+				Destination: &opt.OrbitChannel,
+			},
+			&cli.StringFlag{
+				Name:        "update-url",
+				Usage:       "URL for update server",
+				Value:       "https://tuf.fleetctl.com",
+				Destination: &opt.UpdateURL,
+			},
+			&cli.StringFlag{
+				Name:        "update-roots",
+				Usage:       "Root key JSON metadata for update server (from fleetctl updates roots)",
+				Destination: &opt.UpdateRoots,
+			},
+			&cli.BoolFlag{
+				Name:        "debug",
+				Usage:       "Enable debug logging",
+				Destination: &opt.Debug,
+			},
+		},
+		Action: func(c *cli.Context) error {
+			if opt.FleetURL != "" || opt.EnrollSecret != "" {
+				if opt.FleetURL == "" || opt.EnrollSecret == "" {
+					return errors.New("--enroll-secret and --fleet-url must be provided together")
+				}
+			}
+
+			if opt.Insecure && opt.FleetCertificate != "" {
+				return errors.New("--insecure and --fleet-certificate may not be provided together")
+			}
+
+			switch c.String("type") {
+			case "pkg":
+				return packaging.BuildPkg(opt)
+			case "deb":
+				return packaging.BuildDeb(opt)
+			case "rpm":
+				return packaging.BuildRPM(opt)
+			case "msi":
+				return packaging.BuildMSI(opt)
+			default:
+				return errors.New("type must be one of ('pkg', 'deb', 'rpm', 'msi')")
+			}
+		},
+	}
+}

cmd/fleetctl/package_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/stretchr/testify/require"
+	"os"
+	"testing"
+)
+
+func TestPackage(t *testing.T) {
+
+	// --type is required
+	runAppCheckErr(t, []string{"package", "deb"}, "Required flag \"type\" not set")
+
+	// if you provide -fleet-url & --enroll-secret are required together
+	runAppCheckErr(t, []string{"package", "--type=deb", "--fleet-url=https://localhost:8080"}, "--enroll-secret and --fleet-url must be provided together")
+	runAppCheckErr(t, []string{"package", "--type=deb", "--enroll-secret=foobar"}, "--enroll-secret and --fleet-url must be provided together")
+
+	// --insecure and --fleet-certificate are mutually exclusive
+	runAppCheckErr(t, []string{"package", "--type=deb", "--insecure", "--fleet-certificate=test123"}, "--insecure and --fleet-certificate may not be provided together")
+
+	// run package tests, each should output their respective package type
+	// orbit-osquery_0.0.3_amd64.deb
+	runAppForTest(t, []string{"package", "--type=deb", "--insecure"})
+	info, err := os.Stat("orbit-osquery_0.0.3_amd64.deb")
+	require.NoError(t, err)
+	require.Greater(t, info.Size(), int64(0)) // TODO verify contents
+	// orbit-osquery-0.0.3.x86_64.rpm
+	runAppForTest(t, []string{"package", "--type=rpm", "--insecure"})
+	info, err = os.Stat("orbit-osquery-0.0.3.x86_64.rpm")
+	require.NoError(t, err)
+	require.Greater(t, info.Size(), int64(0)) // TODO verify contents
+	// orbit-osquery_0.0.3.msi
+	//runAppForTest(t, []string{"package", "--type=msi", "--insecure"}) TODO: this is currently failing on Github runners due to permission issues
+	//info, err = os.Stat("orbit-osquery_0.0.3.msi")
+	//require.NoError(t, err)
+	//require.Greater(t, info.Size(), int64(0))
+
+	//runAppForTest(t, []string{"package", "--type=pkg", "--insecure"}) TODO: had a hard time getting xar installed on Ubuntu
+
+}

docs/2-Deploying/4-fleetctl-agent-updates.md

@@ -128,8 +128,12 @@ fleetctl updates roots
 
 This output is _not sensitive_ and will be shared in agent deployments to verify the contents of updates and metadata. Provide the JSON output in the `--update-roots` flag of the [Orbit packager](https://github.com/fleetdm/orbit#packaging):
 
-For example:
 
-```
+### Packaging with Orbit
-go run ./cmd/package --type pkg --enroll-secret=ZB3QfaItRRUjlGo+LctfOMzLTjjyxLdg --fleet-url=https://localhost:8080 --update-url=http://localhost:8000 --update-roots='[{"keytype":"ed25519","scheme":"ed25519","keyid_hash_algorithms":["sha256","sha512"],"keyval":{"public":"9aac6f59f0e3f2f4ef6f789e4eff058af27db3743eaa6a47a228f23e5a1ad4d7"}}]' --insecure
+
-```
+See [Orbit Docs](https://github.com/fleetdm/fleet/blob/main/orbit/README.md) for more details
+
+You can use `fleetctl package` to generate installer packages of Orbit (a bootstrapped OSQuery wrapper) to integrate with your Fleet instance.
+
+For example running `fleetctl package --type deb --fleet-url=<fleet url> --enroll-secret=<enroll secret>` will build a `.deb` installer with everything needed
+to communicate with your fleet instance.

orbit/cmd/package/package.go

@@ -1,142 +0,0 @@
-package main
-
-import (
-	"os"
-	"time"
-
-	"github.com/fleetdm/fleet/v4/orbit/pkg/packaging"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
-	"github.com/urfave/cli/v2"
-)
-
-func main() {
-	var opt packaging.Options
-	log.Logger = log.Output(
-		zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339Nano},
-	)
-	zerolog.SetGlobalLevel(zerolog.InfoLevel)
-
-	app := cli.NewApp()
-	app.Name = "Orbit osquery"
-	app.Usage = "A powered-up, (near) drop-in replacement for osquery"
-	app.Commands = []*cli.Command{}
-	app.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:     "type",
-			Usage:    "Type of package to build",
-			Required: true,
-		},
-		&cli.StringFlag{
-			Name:        "enroll-secret",
-			Usage:       "Enroll secret for authenticating to Fleet server",
-			Destination: &opt.EnrollSecret,
-		},
-		&cli.StringFlag{
-			Name:        "fleet-url",
-			Usage:       "URL (host:port) of Fleet server",
-			Destination: &opt.FleetURL,
-		},
-		&cli.StringFlag{
-			Name:        "fleet-certificate",
-			Usage:       "Path to server cerificate bundle",
-			Destination: &opt.FleetCertificate,
-		},
-		&cli.StringFlag{
-			Name:        "identifier",
-			Usage:       "Identifier for package product",
-			Value:       "com.fleetdm.orbit",
-			Destination: &opt.Identifier,
-		},
-		&cli.StringFlag{
-			Name:        "version",
-			Usage:       "Version for package product",
-			Value:       "0.0.3",
-			Destination: &opt.Version,
-		},
-		&cli.BoolFlag{
-			Name:        "insecure",
-			Usage:       "Disable TLS certificate verification",
-			Destination: &opt.Insecure,
-		},
-		&cli.BoolFlag{
-			Name:        "service",
-			Usage:       "Install orbit/osquery with a persistence service (launchd, systemd, etc.)",
-			Value:       true,
-			Destination: &opt.StartService,
-		},
-		&cli.StringFlag{
-			Name:        "sign-identity",
-			Usage:       "Identity to use for macOS codesigning",
-			Destination: &opt.SignIdentity,
-		},
-		&cli.BoolFlag{
-			Name:        "notarize",
-			Usage:       "Whether to notarize macOS packages",
-			Destination: &opt.Notarize,
-		},
-		&cli.StringFlag{
-			Name:        "osqueryd-channel",
-			Usage:       "Update channel of osqueryd to use",
-			Value:       "stable",
-			Destination: &opt.OsquerydChannel,
-		},
-		&cli.StringFlag{
-			Name:        "orbit-channel",
-			Usage:       "Update channel of Orbit to use",
-			Value:       "stable",
-			Destination: &opt.OrbitChannel,
-		},
-		&cli.StringFlag{
-			Name:        "update-url",
-			Usage:       "URL for update server",
-			Value:       "https://tuf.fleetctl.com",
-			Destination: &opt.UpdateURL,
-		},
-		&cli.StringFlag{
-			Name:        "update-roots",
-			Usage:       "Root key JSON metadata for update server (from fleetctl updates roots)",
-			Destination: &opt.UpdateRoots,
-		},
-		&cli.BoolFlag{
-			Name:        "debug",
-			Usage:       "Enable debug logging",
-			Destination: &opt.Debug,
-		},
-	}
-	app.Before = func(c *cli.Context) error {
-		if opt.Debug {
-			zerolog.SetGlobalLevel(zerolog.DebugLevel)
-		}
-		return nil
-	}
-	app.Action = func(c *cli.Context) error {
-		if opt.FleetURL != "" || opt.EnrollSecret != "" {
-			if opt.FleetURL == "" || opt.EnrollSecret == "" {
-				return errors.New("--enroll-secret and --fleet-url must be provided together")
-			}
-		}
-
-		if opt.Insecure && opt.FleetCertificate != "" {
-			return errors.New("--insecure and --fleet-certificate may not be provided together")
-		}
-
-		switch c.String("type") {
-		case "pkg":
-			return packaging.BuildPkg(opt)
-		case "deb":
-			return packaging.BuildDeb(opt)
-		case "rpm":
-			return packaging.BuildRPM(opt)
-		case "msi":
-			return packaging.BuildMSI(opt)
-		default:
-			return errors.New("type must be one of ('pkg', 'deb', 'rpm', 'msi')")
-		}
-	}
-
-	if err := app.Run(os.Args); err != nil {
-		log.Fatal().Err(err).Msg("package failed")
-	}
-}

orbit/pkg/packaging/linux_shared.go

@@ -18,7 +18,6 @@ import (
 
 func buildNFPM(opt Options, pkger nfpm.Packager) error {
 	// Initialize directories
-
 	tmpDir, err := ioutil.TempDir("", "orbit-package")
 	if err != nil {
 		return errors.Wrap(err, "failed to create temp dir")

orbit/pkg/packaging/macos.go

@@ -22,7 +22,6 @@ import (
 // Linux.
 func BuildPkg(opt Options) error {
 	// Initialize directories
-
 	tmpDir, err := ioutil.TempDir("", "orbit-package")
 	if err != nil {
 		return errors.Wrap(err, "failed to create temp dir")

orbit/pkg/packaging/packaging.go

@@ -1,4 +1,4 @@
-// package packaging provides tools for buildin Orbit installation packages.
+// Package packaging provides tools for building Orbit installation packages.
 package packaging
 
 import (

orbit/pkg/update/filestore/filestore.go

@@ -65,7 +65,7 @@ func (s *fileStore) readData() error {
 		return errors.New("expected file store to be regular file")
 	}
 
-	f, err := os.Open(s.filename)
+	f, err := secure.OpenFile(s.filename, os.O_RDWR|os.O_CREATE, constant.DefaultFileMode)
 	if err != nil {
 		return errors.Wrap(err, "open file store")
 	}