report errors that can occur during file carving (#8972)

related to https://github.com/fleetdm/fleet/issues/8117

changes/8117-file-carving-errors

@@ -0,0 +1 @@
+- Added functionality to report if a carve failed along with its error message.

cmd/fleet/main.go

@@ -69,6 +69,17 @@ func applyDevFlags(cfg *config.FleetConfig) {
 		cfg.Prometheus.BasicAuth.Password = "insecure"
 	}
 
+	cfg.S3 = config.S3Config{
+		Bucket:           "carves-dev",
+		Region:           "minio",
+		Prefix:           "dev-prefix",
+		EndpointURL:      "localhost:9000",
+		AccessKeyID:      "minio",
+		SecretAccessKey:  "minio123!",
+		DisableSSL:       true,
+		ForceS3PathStyle: true,
+	}
+
 	cfg.Packaging.S3 = config.S3Config{
 		Bucket:           "installers-dev",
 		Region:           "minio",

cmd/fleetctl/get.go

@@ -737,16 +737,22 @@ func getCarvesCommand() *cli.Command {
 					completion = "Expired"
 				}
 
+				errored := "no"
+				if c.Error != nil {
+					errored = "yes"
+				}
+
 				data = append(data, []string{
 					strconv.FormatInt(c.ID, 10),
-					c.CreatedAt.Local().String(),
+					c.CreatedAt.String(),
 					c.RequestId,
 					strconv.FormatInt(c.CarveSize, 10),
 					completion,
+					errored,
 				})
 			}
 
-			columns := []string{"id", "created_at", "request_id", "carve_size", "completion"}
+			columns := []string{"id", "created_at", "request_id", "carve_size", "completion", "errored"}
 			printTable(c, columns, data)
 
 			return nil
@@ -789,6 +795,15 @@ func getCarveCommand() *cli.Command {
 				return errors.New("-stdout and -outfile must not be specified together")
 			}
 
+			carve, err := client.GetCarve(id)
+			if err != nil {
+				return err
+			}
+
+			if carve.Error != nil {
+				return errors.New(*carve.Error)
+			}
+
 			if stdout || outFile != "" {
 				out := os.Stdout
 				if outFile != "" {
@@ -812,11 +827,6 @@ func getCarveCommand() *cli.Command {
 				return nil
 			}
 
-			carve, err := client.GetCarve(id)
-			if err != nil {
-				return err
-			}
-
 			if err := printYaml(carve, c.App.Writer); err != nil {
 				return fmt.Errorf("print carve yaml: %w", err)
 			}

cmd/fleetctl/get_test.go

@@ -1460,3 +1460,110 @@ func TestGetAppleMDM(t *testing.T) {
 	expected := `Error: No Apple Push Notification service (APNs) certificate found.`
 	assert.Contains(t, runAppForTest(t, []string{"get", "mdm_apple"}), expected)
 }
+
+func TestGetCarves(t *testing.T) {
+	_, ds := runServerWithMockedDS(t)
+
+	createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z")
+	require.NoError(t, err)
+	ds.ListCarvesFunc = func(ctx context.Context, opts fleet.CarveListOptions) ([]*fleet.CarveMetadata, error) {
+		return []*fleet.CarveMetadata{
+			{
+				HostId:     1,
+				Name:       "foobar",
+				BlockCount: 10,
+				BlockSize:  12,
+				CarveSize:  123,
+				CarveId:    "carve_id_1",
+				RequestId:  "request_id_1",
+				SessionId:  "session_id_1",
+				CreatedAt:  createdAt,
+			},
+			{
+				HostId:     2,
+				Name:       "barfoo",
+				BlockCount: 20,
+				BlockSize:  44,
+				CarveSize:  123,
+				CarveId:    "carve_id_2",
+				RequestId:  "request_id_2",
+				SessionId:  "session_id_2",
+				CreatedAt:  createdAt,
+				Error:      ptr.String("test error"),
+			},
+		}, nil
+	}
+
+	expected := `+----+--------------------------------+--------------+------------+------------+---------+
+| ID |           CREATED AT           |  REQUEST ID  | CARVE SIZE | COMPLETION | ERRORED |
++----+--------------------------------+--------------+------------+------------+---------+
+|  0 | 1999-03-10 02:45:06.371 +0000  | request_id_1 |        123 | 10%        | no      |
+|    | UTC                            |              |            |            |         |
++----+--------------------------------+--------------+------------+------------+---------+
+|  0 | 1999-03-10 02:45:06.371 +0000  | request_id_2 |        123 | 5%         | yes     |
+|    | UTC                            |              |            |            |         |
++----+--------------------------------+--------------+------------+------------+---------+
+`
+	assert.Equal(t, expected, runAppForTest(t, []string{"get", "carves"}))
+}
+
+func TestGetCarve(t *testing.T) {
+	_, ds := runServerWithMockedDS(t)
+
+	createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z")
+	require.NoError(t, err)
+	ds.CarveFunc = func(ctx context.Context, carveID int64) (*fleet.CarveMetadata, error) {
+		return &fleet.CarveMetadata{
+			HostId:     1,
+			Name:       "foobar",
+			BlockCount: 10,
+			BlockSize:  12,
+			CarveSize:  123,
+			CarveId:    "carve_id_1",
+			RequestId:  "request_id_1",
+			SessionId:  "session_id_1",
+			CreatedAt:  createdAt,
+		}, nil
+	}
+
+	expectedOut := `---
+block_count: 10
+block_size: 12
+carve_id: carve_id_1
+carve_size: 123
+created_at: "1999-03-10T02:45:06.371Z"
+error: null
+expired: false
+host_id: 1
+id: 0
+max_block: 0
+name: foobar
+request_id: request_id_1
+session_id: session_id_1
+`
+
+	assert.Equal(t, expectedOut, runAppForTest(t, []string{"get", "carve", "1"}))
+}
+
+func TestGetCarveWithError(t *testing.T) {
+	_, ds := runServerWithMockedDS(t)
+
+	createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z")
+	require.NoError(t, err)
+	ds.CarveFunc = func(ctx context.Context, carveID int64) (*fleet.CarveMetadata, error) {
+		return &fleet.CarveMetadata{
+			HostId:     1,
+			Name:       "foobar",
+			BlockCount: 10,
+			BlockSize:  12,
+			CarveSize:  123,
+			CarveId:    "carve_id_1",
+			RequestId:  "request_id_1",
+			SessionId:  "session_id_1",
+			CreatedAt:  createdAt,
+			Error:      ptr.String("test error"),
+		}, nil
+	}
+
+	runAppCheckErr(t, []string{"get", "carve", "1"}, "test error")
+}

docs/Using-Fleet/REST-API.md

@@ -653,7 +653,8 @@ None.
       "request_id": "fleet_distributed_query_31",
       "session_id": "f73922ed-40a4-4e98-a50a-ccda9d3eb755",
       "expired": false,
-      "max_block": 1
+      "max_block": 1,
+      "error": "S3 multipart carve upload: EntityTooSmall: Your proposed upload is smaller than the minimum allowed object size"
     }
   ]
 }

server/datastore/mysql/carves.go

@@ -11,7 +11,7 @@ import (
 	"github.com/jmoiron/sqlx"
 )
 
-func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) {
+func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fleet.CarveMetadata) (int64, error) {
 	stmt := `INSERT INTO carve_metadata (
 		host_id,
 		created_at,
@@ -21,7 +21,8 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
 		carve_size,
 		carve_id,
 		request_id,
-		session_id
+		session_id,
+		error
 	) VALUES (
 		?,
 		?,
@@ -31,10 +32,11 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
 		?,
 		?,
 		?,
+		?,
 		?
 	)`
 
-	result, err := ds.writer.ExecContext(
+	result, err := writer.ExecContext(
 		ctx,
 		stmt,
 		metadata.HostId,
@@ -46,14 +48,20 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
 		metadata.CarveId,
 		metadata.RequestId,
 		metadata.SessionId,
+		metadata.Error,
 	)
+	if err != nil {
+		return 0, ctxerr.Wrap(ctx, err, "insert carve metadata")
+	}
+	return result.LastInsertId()
+}
+
+func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) {
+	id, err := upsertCarveDB(ctx, ds.writer, metadata)
 	if err != nil {
 		return nil, ctxerr.Wrap(ctx, err, "insert carve metadata")
 	}
-
-	id, _ := result.LastInsertId()
 	metadata.ID = id
-
 	return metadata, nil
 }
 
@@ -67,7 +75,8 @@ func updateCarveDB(ctx context.Context, exec sqlx.ExecerContext, metadata *fleet
 	stmt := `
 		UPDATE carve_metadata SET
 			max_block = ?,
-			expired = ?
+			expired = ?,
+			error = ?
 		WHERE id = ?
 	`
 	_, err := exec.ExecContext(
@@ -75,6 +84,7 @@ func updateCarveDB(ctx context.Context, exec sqlx.ExecerContext, metadata *fleet
 		stmt,
 		metadata.MaxBlock,
 		metadata.Expired,
+		metadata.Error,
 		metadata.ID,
 	)
 	return ctxerr.Wrap(ctx, err, "update carve metadata")
@@ -154,7 +164,8 @@ const carveSelectFields = `
 			request_id,
 			session_id,
 			expired,
-			max_block
+			max_block,
+			error
 `
 
 func (ds *Datastore) Carve(ctx context.Context, carveId int64) (*fleet.CarveMetadata, error) {

server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata.go

@@ -0,0 +1,20 @@
+package tables
+
+import (
+	"database/sql"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20221205112142, Down_20221205112142)
+}
+
+func Up_20221205112142(tx *sql.Tx) error {
+	_, err := tx.Exec("ALTER TABLE `carve_metadata` ADD COLUMN `error` TEXT")
+	return errors.Wrap(err, "adding error column to carve_metadata")
+}
+
+func Down_20221205112142(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata_test.go

@@ -0,0 +1,53 @@
+package tables
+
+import (
+	"database/sql"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20221205112142(t *testing.T) {
+	db := applyUpToPrev(t)
+	query := `
+INSERT INTO carve_metadata
+  (host_id, block_count, block_size, carve_size, carve_id, request_id, session_id)
+VALUES
+  (1, 10, 1000, 10000, "carve_id", "request_id", ?) 
+`
+
+	execNoErr(t, db, "INSERT INTO hosts (hostname, osquery_host_id) VALUES ('foo.example.com', 'foo')")
+	execNoErr(t, db, query, 1)
+	execNoErr(t, db, query, 2)
+
+	// Apply current migration.
+	applyNext(t, db)
+
+	// Okay if we don't provide an error
+	execNoErr(t, db, query, 3)
+	// Insert with an error
+	execNoErr(t, db, `
+INSERT INTO carve_metadata
+  (host_id, block_count, block_size, carve_size, carve_id, request_id, session_id, error)
+VALUES
+  (1, 10, 1000, 10000, "carve_id", "request_id", 4, "made_up_error") 
+`)
+	// Update an existing row to add an error
+	execNoErr(t, db, `UPDATE carve_metadata SET error = "updated_error" WHERE session_id = 3`)
+
+	var storedErr sql.NullString
+	row := db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 3`)
+	err := row.Scan(&storedErr)
+	require.NoError(t, err)
+	require.Equal(t, "updated_error", storedErr.String)
+
+	row = db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 4`)
+	err = row.Scan(&storedErr)
+	require.NoError(t, err)
+	require.Equal(t, "made_up_error", storedErr.String)
+
+	row = db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 1`)
+	err = row.Scan(&storedErr)
+	require.NoError(t, err)
+	require.Equal(t, "", storedErr.String)
+}

server/datastore/mysql/migrations/tables/migration_test.go

@@ -76,8 +76,8 @@ func applyUpToPrev(t *testing.T) *sqlx.DB {
 	}
 }
 
-func execNoErr(t *testing.T, db *sqlx.DB, query string) {
+func execNoErr(t *testing.T, db *sqlx.DB, query string, args ...any) {
-	_, err := db.Exec(query)
+	_, err := db.Exec(query, args...)
 	require.NoError(t, err)
 }
 

server/datastore/s3/carves.go

@@ -14,6 +14,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/config"
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/ptr"
 )
 
 const (
@@ -56,11 +57,24 @@ func (c *CarveStore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
 		Bucket: &c.bucket,
 		Key:    &objectKey,
 	})
+
 	if err != nil {
-		return nil, ctxerr.Wrap(ctx, err, "s3 multipart carve create")
+		// even if we fail to create the multipart upload, we still want to create
+		// the carve in the database and register an error, this way the user can
+		// still fetch the carve and check its status
+		metadata.Error = ptr.String(err.Error())
+		if _, err := c.metadatadb.NewCarve(ctx, metadata); err != nil {
+			return nil, ctxerr.Wrap(ctx, err, "creating carve metadata")
 		}
+		return nil, ctxerr.Wrap(ctx, err, "creating multipart upload")
+	}
+
 	metadata.SessionId = *res.UploadId
-	return c.metadatadb.NewCarve(ctx, metadata)
+	savedMetadata, err := c.metadatadb.NewCarve(ctx, metadata)
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "creating carve metadata")
+	}
+	return savedMetadata, nil
 }
 
 // UpdateCarve updates carve definition in database

server/datastore/s3/carves_test.go

@@ -0,0 +1 @@
+package s3

server/fleet/carves.go

@@ -27,6 +27,8 @@ type CarveMetadata struct {
 	SessionId string `json:"session_id" db:"session_id"`
 	// Expired is whether the carve has "expired" (data has been purged).
 	Expired bool `json:"expired" db:"expired"`
+	// Error is the error message if the carve failed.
+	Error *string `json:"error" db:"error"`
 
 	// MaxBlock is the highest block number currently stored for this carve.
 	// This value is not stored directly, but generated from the carve_blocks

server/service/carves.go

@@ -8,7 +8,9 @@ import (
 
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	hostctx "github.com/fleetdm/fleet/v4/server/contexts/host"
+	"github.com/fleetdm/fleet/v4/server/contexts/logging"
 	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/ptr"
 	"github.com/google/uuid"
 )
 
@@ -290,6 +292,28 @@ func (svc *Service) CarveBlock(ctx context.Context, payload fleet.CarveBlockPayl
 
 	// Request is now authenticated
 
+	if err := svc.validateCarveBlock(payload, carve); err != nil {
+		carve.Error = ptr.String(err.Error())
+		if errRecord := svc.carveStore.UpdateCarve(ctx, carve); err != nil {
+			logging.WithExtras(ctx, "validate_carve_error", errRecord, "carve_id", carve.ID)
+		}
+
+		return ctxerr.Wrap(ctx, err, "validate carve block")
+	}
+
+	if err := svc.carveStore.NewBlock(ctx, carve, payload.BlockId, payload.Data); err != nil {
+		carve.Error = ptr.String(err.Error())
+		if errRecord := svc.carveStore.UpdateCarve(ctx, carve); err != nil {
+			logging.WithExtras(ctx, "record_carve_error", errRecord, "carve_id", carve.ID)
+		}
+
+		return ctxerr.Wrap(ctx, err, "save carve block data")
+	}
+
+	return nil
+}
+
+func (svc *Service) validateCarveBlock(payload fleet.CarveBlockPayload, carve *fleet.CarveMetadata) error {
 	if payload.BlockId > carve.BlockCount-1 {
 		return fmt.Errorf("block_id exceeds expected max (%d): %d", carve.BlockCount-1, payload.BlockId)
 	}
@@ -302,9 +326,5 @@ func (svc *Service) CarveBlock(ctx context.Context, payload fleet.CarveBlockPayl
 		return fmt.Errorf("exceeded declared block size %d: %d", carve.BlockSize, len(payload.Data))
 	}
 
-	if err := svc.carveStore.NewBlock(ctx, carve, payload.BlockId, payload.Data); err != nil {
-		return ctxerr.Wrap(ctx, err, "save block data")
-	}
-
 	return nil
 }

server/service/carves_test.go

@@ -459,6 +459,11 @@ func TestCarveCarveBlockBlockCountExceedError(t *testing.T) {
 		assert.Equal(t, metadata.SessionId, sessionId)
 		return metadata, nil
 	}
+	ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
+		assert.NotNil(t, carve.Error)
+		assert.Equal(t, *carve.Error, "block_id exceeds expected max (22): 23")
+		return nil
+	}
 
 	payload := fleet.CarveBlockPayload{
 		Data:      []byte("this is the carve data :)"),
@@ -490,6 +495,11 @@ func TestCarveCarveBlockBlockCountMatchError(t *testing.T) {
 		assert.Equal(t, metadata.SessionId, sessionId)
 		return metadata, nil
 	}
+	ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
+		assert.NotNil(t, carve.Error)
+		assert.Equal(t, *carve.Error, "block_id does not match expected block (4): 7")
+		return nil
+	}
 
 	payload := fleet.CarveBlockPayload{
 		Data:      []byte("this is the carve data :)"),
@@ -521,6 +531,11 @@ func TestCarveCarveBlockBlockSizeError(t *testing.T) {
 		assert.Equal(t, metadata.SessionId, sessionId)
 		return metadata, nil
 	}
+	ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
+		assert.NotNil(t, carve.Error)
+		assert.Equal(t, *carve.Error, "exceeded declared block size 16: 37")
+		return nil
+	}
 
 	payload := fleet.CarveBlockPayload{
 		Data:      []byte("this is the carve data :) TOO LONG!!!"),
@@ -555,6 +570,11 @@ func TestCarveCarveBlockNewBlockError(t *testing.T) {
 	ms.NewBlockFunc = func(ctx context.Context, carve *fleet.CarveMetadata, blockId int64, data []byte) error {
 		return errors.New("kaboom!")
 	}
+	ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
+		assert.NotNil(t, carve.Error)
+		assert.Equal(t, *carve.Error, "kaboom!")
+		return nil
+	}
 
 	payload := fleet.CarveBlockPayload{
 		Data:      []byte("this is the carve data :)"),

server/service/integration_core_test.go

@@ -5195,6 +5195,12 @@ func (s *integrationTestSuite) TestCarve() {
 		Data:      []byte("p1."),
 	}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
 
+	checkCarveError := func(id uint, err string) {
+		var getResp getCarveResponse
+		s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/carves/%d", id), nil, http.StatusOK, &getResp)
+		require.Equal(t, err, *getResp.Carve.Error)
+	}
+
 	// sending a block with unexpected block id (expects 0, got 1)
 	s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{
 		BlockId:   1,
@@ -5202,6 +5208,7 @@ func (s *integrationTestSuite) TestCarve() {
 		RequestId: "r1",
 		Data:      []byte("p1."),
 	}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
+	checkCarveError(1, "block_id does not match expected block (0): 1")
 
 	// sending a block with valid payload, block 0
 	s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{
@@ -5230,6 +5237,7 @@ func (s *integrationTestSuite) TestCarve() {
 		RequestId: "r1",
 		Data:      []byte("p2."),
 	}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
+	checkCarveError(1, "block_id does not match expected block (2): 1")
 
 	// sending final block with too many bytes
 	blockResp = carveBlockResponse{}
@@ -5239,6 +5247,7 @@ func (s *integrationTestSuite) TestCarve() {
 		RequestId: "r1",
 		Data:      []byte("p3extra"),
 	}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
+	checkCarveError(1, "exceeded declared block size 3: 7")
 
 	// sending actual final block
 	blockResp = carveBlockResponse{}
@@ -5258,6 +5267,7 @@ func (s *integrationTestSuite) TestCarve() {
 		RequestId: "r1",
 		Data:      []byte("p4."),
 	}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
+	checkCarveError(1, "block_id exceeds expected max (2): 3")
 }
 
 func (s *integrationTestSuite) TestPasswordReset() {