empayre/fleet
14
0
Fork 0
mirror of https://github.com/empayre/fleet.git synced 2024-11-06 00:45:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

report errors that can occur during file carving (#8972)

Browse Source 
related to https://github.com/fleetdm/fleet/issues/8117
This commit is contained in:
Roberto Dip 2022-12-09 13:21:30 -03:00 committed by GitHub
parent 71dbb71df4
commit e68535d468
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 308 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changes/8117-file-carving-errors Normal file
View File

@ -0,0 +1 @@
- Added functionality to report if a carve failed along with its error message.

11
cmd/fleet/main.go
View File

@ -69,6 +69,17 @@ func applyDevFlags(cfg *config.FleetConfig) {
cfg.Prometheus.BasicAuth.Password = "insecure"
}
cfg.S3 = config.S3Config{
Bucket: "carves-dev",
Region: "minio",
Prefix: "dev-prefix",
EndpointURL: "localhost:9000",
AccessKeyID: "minio",
SecretAccessKey: "minio123!",
DisableSSL: true,
ForceS3PathStyle: true,
}
cfg.Packaging.S3 = config.S3Config{
Bucket: "installers-dev",
Region: "minio",

24
cmd/fleetctl/get.go
View File

@ -737,16 +737,22 @@ func getCarvesCommand() *cli.Command {
completion = "Expired"
}
errored := "no"
if c.Error != nil {
errored = "yes"
}
data = append(data, []string{
strconv.FormatInt(c.ID, 10),
c.CreatedAt.Local().String(),
c.CreatedAt.String(),
c.RequestId,
strconv.FormatInt(c.CarveSize, 10),
completion,
errored,
})
}
columns := []string{"id", "created_at", "request_id", "carve_size", "completion"}
columns := []string{"id", "created_at", "request_id", "carve_size", "completion", "errored"}
printTable(c, columns, data)
return nil
@ -789,6 +795,15 @@ func getCarveCommand() *cli.Command {
return errors.New("-stdout and -outfile must not be specified together")
}
carve, err := client.GetCarve(id)
if err != nil {
return err
}
if carve.Error != nil {
return errors.New(*carve.Error)
}
if stdout || outFile != "" {
out := os.Stdout
if outFile != "" {
@ -812,11 +827,6 @@ func getCarveCommand() *cli.Command {
return nil
}
carve, err := client.GetCarve(id)
if err != nil {
return err
}
if err := printYaml(carve, c.App.Writer); err != nil {
return fmt.Errorf("print carve yaml: %w", err)
}

107
cmd/fleetctl/get_test.go
View File

@ -1460,3 +1460,110 @@ func TestGetAppleMDM(t *testing.T) {
expected := `Error: No Apple Push Notification service (APNs) certificate found.`
assert.Contains(t, runAppForTest(t, []string{"get", "mdm_apple"}), expected)
}
func TestGetCarves(t *testing.T) {
_, ds := runServerWithMockedDS(t)
createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z")
require.NoError(t, err)
ds.ListCarvesFunc = func(ctx context.Context, opts fleet.CarveListOptions) ([]*fleet.CarveMetadata, error) {
return []*fleet.CarveMetadata{
{
HostId: 1,
Name: "foobar",
BlockCount: 10,
BlockSize: 12,
CarveSize: 123,
CarveId: "carve_id_1",
RequestId: "request_id_1",
SessionId: "session_id_1",
CreatedAt: createdAt,
},
{
HostId: 2,
Name: "barfoo",
BlockCount: 20,
BlockSize: 44,
CarveSize: 123,
CarveId: "carve_id_2",
RequestId: "request_id_2",
SessionId: "session_id_2",
CreatedAt: createdAt,
Error: ptr.String("test error"),
},
}, nil
}
expected := `+----+--------------------------------+--------------+------------+------------+---------+
| ID | CREATED AT | REQUEST ID | CARVE SIZE | COMPLETION | ERRORED |
+----+--------------------------------+--------------+------------+------------+---------+
| 0 | 1999-03-10 02:45:06.371 +0000 | request_id_1 | 123 | 10% | no |
| | UTC | | | | |
+----+--------------------------------+--------------+------------+------------+---------+
| 0 | 1999-03-10 02:45:06.371 +0000 | request_id_2 | 123 | 5% | yes |
| | UTC | | | | |
+----+--------------------------------+--------------+------------+------------+---------+
`
assert.Equal(t, expected, runAppForTest(t, []string{"get", "carves"}))
}
func TestGetCarve(t *testing.T) {
_, ds := runServerWithMockedDS(t)
createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z")
require.NoError(t, err)
ds.CarveFunc = func(ctx context.Context, carveID int64) (*fleet.CarveMetadata, error) {
return &fleet.CarveMetadata{
HostId: 1,
Name: "foobar",
BlockCount: 10,
BlockSize: 12,
CarveSize: 123,
CarveId: "carve_id_1",
RequestId: "request_id_1",
SessionId: "session_id_1",
CreatedAt: createdAt,
}, nil
}
expectedOut := `---
block_count: 10
block_size: 12
carve_id: carve_id_1
carve_size: 123
created_at: "1999-03-10T02:45:06.371Z"
error: null
expired: false
host_id: 1
id: 0
max_block: 0
name: foobar
request_id: request_id_1
session_id: session_id_1
`
assert.Equal(t, expectedOut, runAppForTest(t, []string{"get", "carve", "1"}))
}
func TestGetCarveWithError(t *testing.T) {
_, ds := runServerWithMockedDS(t)
createdAt, err := time.Parse(time.RFC3339, "1999-03-10T02:45:06.371Z")
require.NoError(t, err)
ds.CarveFunc = func(ctx context.Context, carveID int64) (*fleet.CarveMetadata, error) {
return &fleet.CarveMetadata{
HostId: 1,
Name: "foobar",
BlockCount: 10,
BlockSize: 12,
CarveSize: 123,
CarveId: "carve_id_1",
RequestId: "request_id_1",
SessionId: "session_id_1",
CreatedAt: createdAt,
Error: ptr.String("test error"),
}, nil
}
runAppCheckErr(t, []string{"get", "carve", "1"}, "test error")
}

3
docs/Using-Fleet/REST-API.md
View File

@ -653,7 +653,8 @@ None.
"request_id": "fleet_distributed_query_31",
"session_id": "f73922ed-40a4-4e98-a50a-ccda9d3eb755",
"expired": false,
"max_block": 1
"max_block": 1,
"error": "S3 multipart carve upload: EntityTooSmall: Your proposed upload is smaller than the minimum allowed object size"
}
]
}

27
server/datastore/mysql/carves.go
View File

@ -11,7 +11,7 @@ import (
"github.com/jmoiron/sqlx"
)
func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) {
func upsertCarveDB(ctx context.Context, writer sqlx.ExecerContext, metadata *fleet.CarveMetadata) (int64, error) {
stmt := `INSERT INTO carve_metadata (
host_id,
created_at,
@ -21,7 +21,8 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
carve_size,
carve_id,
request_id,
session_id
session_id,
error
) VALUES (
?,
?,
@ -31,10 +32,11 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
?,
?,
?,
?,
?
)`
result, err := ds.writer.ExecContext(
result, err := writer.ExecContext(
ctx,
stmt,
metadata.HostId,
@ -46,14 +48,20 @@ func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
metadata.CarveId,
metadata.RequestId,
metadata.SessionId,
metadata.Error,
)
if err != nil {
return 0, ctxerr.Wrap(ctx, err, "insert carve metadata")
}
return result.LastInsertId()
}
func (ds *Datastore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) {
id, err := upsertCarveDB(ctx, ds.writer, metadata)
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "insert carve metadata")
}
id, _ := result.LastInsertId()
metadata.ID = id
return metadata, nil
}
@ -67,7 +75,8 @@ func updateCarveDB(ctx context.Context, exec sqlx.ExecerContext, metadata *fleet
stmt := `
UPDATE carve_metadata SET
max_block = ?,
expired = ?
expired = ?,
error = ?
WHERE id = ?
`
_, err := exec.ExecContext(
@ -75,6 +84,7 @@ func updateCarveDB(ctx context.Context, exec sqlx.ExecerContext, metadata *fleet
stmt,
metadata.MaxBlock,
metadata.Expired,
metadata.Error,
metadata.ID,
)
return ctxerr.Wrap(ctx, err, "update carve metadata")
@ -154,7 +164,8 @@ const carveSelectFields = `
request_id,
session_id,
expired,
max_block
max_block,
error
`
func (ds *Datastore) Carve(ctx context.Context, carveId int64) (*fleet.CarveMetadata, error) {

20
server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata.go Normal file
View File

@ -0,0 +1,20 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20221205112142, Down_20221205112142)
}
func Up_20221205112142(tx *sql.Tx) error {
_, err := tx.Exec("ALTER TABLE `carve_metadata` ADD COLUMN `error` TEXT")
return errors.Wrap(err, "adding error column to carve_metadata")
}
func Down_20221205112142(tx *sql.Tx) error {
return nil
}

53
server/datastore/mysql/migrations/tables/20221205112142_AddErrorToCarveMetadata_test.go Normal file
View File

@ -0,0 +1,53 @@
package tables
import (
"database/sql"
"testing"
"github.com/stretchr/testify/require"
)
func TestUp_20221205112142(t *testing.T) {
db := applyUpToPrev(t)
query := `
INSERT INTO carve_metadata
(host_id, block_count, block_size, carve_size, carve_id, request_id, session_id)
VALUES
(1, 10, 1000, 10000, "carve_id", "request_id", ?)
`
execNoErr(t, db, "INSERT INTO hosts (hostname, osquery_host_id) VALUES ('foo.example.com', 'foo')")
execNoErr(t, db, query, 1)
execNoErr(t, db, query, 2)
// Apply current migration.
applyNext(t, db)
// Okay if we don't provide an error
execNoErr(t, db, query, 3)
// Insert with an error
execNoErr(t, db, `
INSERT INTO carve_metadata
(host_id, block_count, block_size, carve_size, carve_id, request_id, session_id, error)
VALUES
(1, 10, 1000, 10000, "carve_id", "request_id", 4, "made_up_error")
`)
// Update an existing row to add an error
execNoErr(t, db, `UPDATE carve_metadata SET error = "updated_error" WHERE session_id = 3`)
var storedErr sql.NullString
row := db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 3`)
err := row.Scan(&storedErr)
require.NoError(t, err)
require.Equal(t, "updated_error", storedErr.String)
row = db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 4`)
err = row.Scan(&storedErr)
require.NoError(t, err)
require.Equal(t, "made_up_error", storedErr.String)
row = db.QueryRow(`SELECT error FROM carve_metadata WHERE session_id = 1`)
err = row.Scan(&storedErr)
require.NoError(t, err)
require.Equal(t, "", storedErr.String)
}

4
server/datastore/mysql/migrations/tables/migration_test.go
View File

@ -76,8 +76,8 @@ func applyUpToPrev(t *testing.T) *sqlx.DB {
}
}
func execNoErr(t *testing.T, db *sqlx.DB, query string) {
_, err := db.Exec(query)
func execNoErr(t *testing.T, db *sqlx.DB, query string, args ...any) {
_, err := db.Exec(query, args...)
require.NoError(t, err)
}

5
server/datastore/mysql/schema.sql
View File

File diff suppressed because one or more lines are too long

18
server/datastore/s3/carves.go
View File

@ -14,6 +14,7 @@ import (
"github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
)
const (
@ -56,11 +57,24 @@ func (c *CarveStore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata
Bucket: &c.bucket,
Key: &objectKey,
})
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "s3 multipart carve create")
// even if we fail to create the multipart upload, we still want to create
// the carve in the database and register an error, this way the user can
// still fetch the carve and check its status
metadata.Error = ptr.String(err.Error())
if _, err := c.metadatadb.NewCarve(ctx, metadata); err != nil {
return nil, ctxerr.Wrap(ctx, err, "creating carve metadata")
}
return nil, ctxerr.Wrap(ctx, err, "creating multipart upload")
}
metadata.SessionId = *res.UploadId
return c.metadatadb.NewCarve(ctx, metadata)
savedMetadata, err := c.metadatadb.NewCarve(ctx, metadata)
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "creating carve metadata")
}
return savedMetadata, nil
}
// UpdateCarve updates carve definition in database

1
server/datastore/s3/carves_test.go Normal file
View File

@ -0,0 +1 @@
package s3

2
server/fleet/carves.go
View File

@ -27,6 +27,8 @@ type CarveMetadata struct {
SessionId string `json:"session_id" db:"session_id"`
// Expired is whether the carve has "expired" (data has been purged).
Expired bool `json:"expired" db:"expired"`
// Error is the error message if the carve failed.
Error *string `json:"error" db:"error"`
// MaxBlock is the highest block number currently stored for this carve.
// This value is not stored directly, but generated from the carve_blocks

28
server/service/carves.go
View File

@ -8,7 +8,9 @@ import (
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
hostctx "github.com/fleetdm/fleet/v4/server/contexts/host"
"github.com/fleetdm/fleet/v4/server/contexts/logging"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/google/uuid"
)
@ -290,6 +292,28 @@ func (svc *Service) CarveBlock(ctx context.Context, payload fleet.CarveBlockPayl
// Request is now authenticated
if err := svc.validateCarveBlock(payload, carve); err != nil {
carve.Error = ptr.String(err.Error())
if errRecord := svc.carveStore.UpdateCarve(ctx, carve); err != nil {
logging.WithExtras(ctx, "validate_carve_error", errRecord, "carve_id", carve.ID)
}
return ctxerr.Wrap(ctx, err, "validate carve block")
}
if err := svc.carveStore.NewBlock(ctx, carve, payload.BlockId, payload.Data); err != nil {
carve.Error = ptr.String(err.Error())
if errRecord := svc.carveStore.UpdateCarve(ctx, carve); err != nil {
logging.WithExtras(ctx, "record_carve_error", errRecord, "carve_id", carve.ID)
}
return ctxerr.Wrap(ctx, err, "save carve block data")
}
return nil
}
func (svc *Service) validateCarveBlock(payload fleet.CarveBlockPayload, carve *fleet.CarveMetadata) error {
if payload.BlockId > carve.BlockCount-1 {
return fmt.Errorf("block_id exceeds expected max (%d): %d", carve.BlockCount-1, payload.BlockId)
}
@ -302,9 +326,5 @@ func (svc *Service) CarveBlock(ctx context.Context, payload fleet.CarveBlockPayl
return fmt.Errorf("exceeded declared block size %d: %d", carve.BlockSize, len(payload.Data))
}
if err := svc.carveStore.NewBlock(ctx, carve, payload.BlockId, payload.Data); err != nil {
return ctxerr.Wrap(ctx, err, "save block data")
}
return nil
}

20
server/service/carves_test.go
View File

@ -459,6 +459,11 @@ func TestCarveCarveBlockBlockCountExceedError(t *testing.T) {
assert.Equal(t, metadata.SessionId, sessionId)
return metadata, nil
}
ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
assert.NotNil(t, carve.Error)
assert.Equal(t, *carve.Error, "block_id exceeds expected max (22): 23")
return nil
}
payload := fleet.CarveBlockPayload{
Data: []byte("this is the carve data :)"),
@ -490,6 +495,11 @@ func TestCarveCarveBlockBlockCountMatchError(t *testing.T) {
assert.Equal(t, metadata.SessionId, sessionId)
return metadata, nil
}
ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
assert.NotNil(t, carve.Error)
assert.Equal(t, *carve.Error, "block_id does not match expected block (4): 7")
return nil
}
payload := fleet.CarveBlockPayload{
Data: []byte("this is the carve data :)"),
@ -521,6 +531,11 @@ func TestCarveCarveBlockBlockSizeError(t *testing.T) {
assert.Equal(t, metadata.SessionId, sessionId)
return metadata, nil
}
ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
assert.NotNil(t, carve.Error)
assert.Equal(t, *carve.Error, "exceeded declared block size 16: 37")
return nil
}
payload := fleet.CarveBlockPayload{
Data: []byte("this is the carve data :) TOO LONG!!!"),
@ -555,6 +570,11 @@ func TestCarveCarveBlockNewBlockError(t *testing.T) {
ms.NewBlockFunc = func(ctx context.Context, carve *fleet.CarveMetadata, blockId int64, data []byte) error {
return errors.New("kaboom!")
}
ms.UpdateCarveFunc = func(ctx context.Context, carve *fleet.CarveMetadata) error {
assert.NotNil(t, carve.Error)
assert.Equal(t, *carve.Error, "kaboom!")
return nil
}
payload := fleet.CarveBlockPayload{
Data: []byte("this is the carve data :)"),

10
server/service/integration_core_test.go
View File

@ -5195,6 +5195,12 @@ func (s *integrationTestSuite) TestCarve() {
Data: []byte("p1."),
}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
checkCarveError := func(id uint, err string) {
var getResp getCarveResponse
s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/carves/%d", id), nil, http.StatusOK, &getResp)
require.Equal(t, err, *getResp.Carve.Error)
}
// sending a block with unexpected block id (expects 0, got 1)
s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{
BlockId: 1,
@ -5202,6 +5208,7 @@ func (s *integrationTestSuite) TestCarve() {
RequestId: "r1",
Data: []byte("p1."),
}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
checkCarveError(1, "block_id does not match expected block (0): 1")
// sending a block with valid payload, block 0
s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{
@ -5230,6 +5237,7 @@ func (s *integrationTestSuite) TestCarve() {
RequestId: "r1",
Data: []byte("p2."),
}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
checkCarveError(1, "block_id does not match expected block (2): 1")
// sending final block with too many bytes
blockResp = carveBlockResponse{}
@ -5239,6 +5247,7 @@ func (s *integrationTestSuite) TestCarve() {
RequestId: "r1",
Data: []byte("p3extra"),
}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
checkCarveError(1, "exceeded declared block size 3: 7")
// sending actual final block
blockResp = carveBlockResponse{}
@ -5258,6 +5267,7 @@ func (s *integrationTestSuite) TestCarve() {
RequestId: "r1",
Data: []byte("p4."),
}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
checkCarveError(1, "block_id exceeds expected max (2): 3")
}
func (s *integrationTestSuite) TestPasswordReset() {