fleet/server/service/client_carves.go
Matteo Piano c89cd370d5
Add AWS S3 as file carving backend (#126)
This adds the option to set up an S3 bucket as the storage backend for file carving (partially solving #111).

It works by using the multipart upload capabilities of S3 to maintain compatibility with the "upload in blocks" protocol that osquery uses. It does this basically replacing the carve_blocks table while still maintaining the metadata in the original place (it would probably be possible to rely completely on S3 by using object tagging at the cost of listing performance). To make this pluggable, I created a new field in the service struct dedicated to the CarveStore which, if no configuration for S3 is set up will be just a reference to the standard datastore, otherwise it will point to the S3 one (effectively this separation will allow in the future to add more backends).
2020-12-16 09:16:55 -08:00

192 lines
4.5 KiB
Go

package service
import (
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/fleetdm/fleet/server/kolide"
"github.com/pkg/errors"
)
// ListCarves lists the file carving sessions
func (c *Client) ListCarves(opt kolide.CarveListOptions) ([]*kolide.CarveMetadata, error) {
endpoint := "/api/v1/kolide/carves"
rawQuery := ""
if opt.Expired {
rawQuery = "expired=1"
}
response, err := c.AuthenticatedDo("GET", endpoint, rawQuery, nil)
if err != nil {
return nil, errors.Wrap(err, "GET /api/v1/kolide/carves")
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, errors.Errorf(
"list carves received status %d %s",
response.StatusCode,
extractServerErrorText(response.Body),
)
}
var responseBody listCarvesResponse
err = json.NewDecoder(response.Body).Decode(&responseBody)
if err != nil {
return nil, errors.Wrap(err, "decode get carves response")
}
if responseBody.Err != nil {
return nil, errors.Errorf("get carves: %s", responseBody.Err)
}
carves := []*kolide.CarveMetadata{}
for _, carve := range responseBody.Carves {
c := carve
carves = append(carves, &c)
}
return carves, nil
}
func (c *Client) GetCarve(carveId int64) (*kolide.CarveMetadata, error) {
endpoint := fmt.Sprintf("/api/v1/kolide/carves/%d", carveId)
response, err := c.AuthenticatedDo("GET", endpoint, "", nil)
if err != nil {
return nil, errors.Wrap(err, "GET "+endpoint)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, errors.Errorf(
"get carve received status %d %s",
response.StatusCode,
extractServerErrorText(response.Body),
)
}
var responseBody getCarveResponse
err = json.NewDecoder(response.Body).Decode(&responseBody)
if err != nil {
return nil, errors.Wrap(err, "decode carve response")
}
if responseBody.Err != nil {
return nil, errors.Errorf("get carve: %s", responseBody.Err)
}
return &responseBody.Carve, nil
}
func (c *Client) getCarveBlock(carveId, blockId int64) ([]byte, error) {
path := fmt.Sprintf(
"/api/v1/kolide/carves/%d/block/%d",
carveId,
blockId,
)
response, err := c.AuthenticatedDo("GET", path, "", nil)
if err != nil {
return nil, errors.Wrapf(err, "GET %s", path)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, errors.Errorf(
"get carve block received status %d: %s",
response.StatusCode,
extractServerErrorText(response.Body),
)
}
var responseBody getCarveBlockResponse
err = json.NewDecoder(response.Body).Decode(&responseBody)
if err != nil {
return nil, errors.Wrap(err, "decode get carve block response")
}
if responseBody.Err != nil {
return nil, errors.Errorf("get carve block: %s", responseBody.Err)
}
return responseBody.Data, nil
}
type carveReader struct {
carve kolide.CarveMetadata
bytesRead int64
curBlock int64
buffer []byte
client *Client
}
func newCarveReader(carve kolide.CarveMetadata, client *Client) *carveReader {
return &carveReader{
carve: carve,
client: client,
bytesRead: 0,
curBlock: 0,
}
}
func (r *carveReader) Read(p []byte) (n int, err error) {
if len(p) == 0 {
return 0, nil
}
if r.bytesRead >= r.carve.CarveSize {
return 0, io.EOF
}
// Load data from API if necessary
if len(r.buffer) == 0 {
var err error
r.buffer, err = r.client.getCarveBlock(r.carve.ID, r.curBlock)
if err != nil {
return 0, errors.Wrapf(err, "get block %d", r.curBlock)
}
r.curBlock++
}
// Calculate length we can copy
copyLen := len(p)
if copyLen > len(r.buffer) {
copyLen = len(r.buffer)
}
// Perform copy and clear copied contents from buffer
copy(p, r.buffer[:copyLen])
r.buffer = r.buffer[copyLen:]
r.bytesRead += int64(copyLen)
return copyLen, nil
}
// DownloadCarve creates a Reader downloading a carve (by ID)
func (c *Client) DownloadCarve(id int64) (io.Reader, error) {
path := fmt.Sprintf("/api/v1/kolide/carves/%d", id)
response, err := c.AuthenticatedDo("GET", path, "", nil)
if err != nil {
return nil, errors.Wrapf(err, "GET %s", path)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return nil, errors.Errorf(
"download carve received status %d: %s",
response.StatusCode,
extractServerErrorText(response.Body),
)
}
var responseBody getCarveResponse
err = json.NewDecoder(response.Body).Decode(&responseBody)
if err != nil {
return nil, errors.Wrap(err, "decode get carve by name response")
}
if responseBody.Err != nil {
return nil, errors.Errorf("get carve by name: %s", responseBody.Err)
}
reader := newCarveReader(responseBody.Carve, c)
return reader, nil
}