add module to enable bringing your own bucket file carving destination (#15206)

This PR adds support for a new terraform module that will make it easy
to configure Fleet instances for S3 backend for file carving results.
Its intended to be applied in two phases.

1) apply target-account which will provision the s3 bucket, IAM role and
policy permissions
2) apply carve on the fleet instance, bootstrapping environment
variables for Fleet server & attaching the IAM policy.

# Checklist for submitter
- [X] Manual QA for all new/changed functionality

terraform/addons/byo-file-carving/carving/.header.md

@@ -0,0 +1,16 @@
+# S3 File Carving backend
+
+This module creates the necessary IAM role for Fleet to attach when it's running in server mode.
+
+It also exports the `fleet_extra_environment_variables` to configure Fleet server to use S3 as the backing carve results store.
+
+Usage typically looks like:
+
+```terraform
+fleet_config = {
+  extra_environment_variables = merge(
+    local.extra_environment_variables,
+    module.carving.fleet_extra_environment_variables 
+  )
+}
+```

terraform/addons/byo-file-carving/carving/.terraform-docs.yml

@@ -0,0 +1 @@
+header-from: .header.md

terraform/addons/byo-file-carving/carving/README.md

@@ -0,0 +1,53 @@
+# S3 File Carving backend
+
+This module creates the necessary IAM role for Fleet to attach when it's running in server mode.
+
+It also exports the `fleet_extra_environment_variables` to configure Fleet server to use S3 as the backing carve results store.
+
+Usage typically looks like:
+
+```terraform
+fleet_config = {
+  extra_environment_variables = merge(
+    local.extra_environment_variables,
+    module.carving.fleet_extra_environment_variables
+  )
+}
+```
+
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_iam_policy.fleet-assume-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy_document.fleet-assume-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_iam_role_arn"></a> [iam\_role\_arn](#input\_iam\_role\_arn) | IAM Role ARN to assume into for file carving uploads to S3 | `string` | n/a | yes |
+| <a name="input_s3_bucket_name"></a> [s3\_bucket\_name](#input\_s3\_bucket\_name) | The S3 bucket for carve results to be written to | `string` | n/a | yes |
+| <a name="input_s3_bucket_region"></a> [s3\_bucket\_region](#input\_s3\_bucket\_region) | The S3 bucket region | `string` | n/a | yes |
+| <a name="input_s3_carve_prefix"></a> [s3\_carve\_prefix](#input\_s3\_carve\_prefix) | The S3 object prefix to use when storing carve results | `string` | `""` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_fleet_extra_environment_variables"></a> [fleet\_extra\_environment\_variables](#output\_fleet\_extra\_environment\_variables) | n/a |
+| <a name="output_fleet_extra_iam_policies"></a> [fleet\_extra\_iam\_policies](#output\_fleet\_extra\_iam\_policies) | n/a |

terraform/addons/byo-file-carving/carving/iam.tf

@@ -0,0 +1,11 @@
+data "aws_iam_policy_document" "fleet-assume-role" {
+  statement {
+    effect    = "Allow"
+    actions   = ["sts:AssumeRole"]
+    resources = [var.iam_role_arn]
+  }
+}
+
+resource "aws_iam_policy" "fleet-assume-role" {
+  policy = data.aws_iam_policy_document.fleet-assume-role.json
+}

terraform/addons/byo-file-carving/carving/outputs.tf

@@ -0,0 +1,14 @@
+output "fleet_extra_environment_variables" {
+  value = {
+    FLEET_S3_STS_ASSUME_ROLE_ARN = var.iam_role_arn
+    FLEET_S3_BUCKET              = var.s3_bucket_name
+    FLEET_S3_REGION              = var.s3_bucket_region
+    FLEET_S3_PREFIX              = var.s3_carve_prefix
+  }
+}
+
+output "fleet_extra_iam_policies" {
+  value = [
+    aws_iam_policy.fleet-assume-role.arn
+  ]
+}

terraform/addons/byo-file-carving/carving/variables.tf

@@ -0,0 +1,20 @@
+variable "iam_role_arn" {
+  type        = string
+  description = "IAM Role ARN to assume into for file carving uploads to S3"
+}
+
+variable "s3_bucket_name" {
+  type =  string
+  description = "The S3 bucket for carve results to be written to"
+}
+
+variable "s3_bucket_region" {
+  type = string
+  description = "The S3 bucket region"
+}
+
+variable "s3_carve_prefix" {
+  type = string
+  description = "The S3 object prefix to use when storing carve results"
+  default = ""
+}

terraform/addons/byo-file-carving/target-account/.header.md

@@ -0,0 +1,25 @@
+# AWS S3 File Carving Infrastructure
+
+This Terraform configuration sets up the necessary resources for a secure file carving infrastructure in AWS. File carving is a significant capability for security and forensic analysis, enabling organizations to extract and analyze the content of files from their endpoints.
+
+## Overview of Resources
+
+The resources configured in this Terraform script include:
+
+- **AWS Key Management Service (KMS) Key**: A customer-managed KMS key is created to provide server-side encryption for the S3 bucket where carved files will be stored. The policy attached to this key grants full KMS permissions to the AWS account's root user.
+
+- **Amazon S3 Bucket**: An S3 bucket is provisioned to act as the central repository for storing the results of the file carving process. The bucket is named according to the provided variable `var.bucket_name`.
+
+- **S3 Bucket Server-Side Encryption Configuration**: This resource configures server-side encryption for the S3 bucket, specifying the custom-created KMS key as the master key for encrypting objects stored in the bucket.
+
+- **IAM Policy**: An IAM policy is created to enable specific access to the S3 bucket. This policy is defined via a detailed policy document which grants permissions to perform various actions essential for managing the file carving process. Actions include object retrieval (`GetObject*`), object creation (`PutObject*`), listing the bucket (`ListBucket*`), and managing multipart uploads. It also allows for certain KMS actions necessary for encrypting and decrypting the stored data.
+
+- **IAM Role**: An IAM role (`aws_iam_role`) is provisioned with a trust relationship policy that permits an external entity, specified by `var.fleet_iam_role_arn`, to assume the role. This allows secure access to the S3 bucket and KMS key based on assuming roles across AWS accounts or services.
+
+- **IAM Role Policy Attachment**: This attachment links the previously created IAM policy to the IAM role, ensuring that the permissions are in effect when the role is assumed by the external entity.
+
+## Usage
+
+To use this Terraform configuration, ensure that you have Terraform installed and configured with the necessary AWS credentials. You should define the `bucket_name` and `fleet_iam_role_arn` variables according to your organization's requirements before applying the Terraform plan.
+
+This infrastructure enables secure storage and access for file carving results, facilitating forensic analysis and the capability to respond to security incidents effectively.

terraform/addons/byo-file-carving/target-account/.terraform-docs.yml

@@ -0,0 +1 @@
+header-from: .header.md

terraform/addons/byo-file-carving/target-account/README.md

@@ -0,0 +1,69 @@
+# AWS S3 File Carving Infrastructure
+
+This Terraform configuration sets up the necessary resources for a secure file carving infrastructure in AWS. File carving is a significant capability for security and forensic analysis, enabling organizations to extract and analyze the content of files from their endpoints.
+
+## Overview of Resources
+
+The resources configured in this Terraform script include:
+
+- **AWS Key Management Service (KMS) Key**: A customer-managed KMS key is created to provide server-side encryption for the S3 bucket where carved files will be stored. The policy attached to this key grants full KMS permissions to the AWS account's root user.
+
+- **Amazon S3 Bucket**: An S3 bucket is provisioned to act as the central repository for storing the results of the file carving process. The bucket is named according to the provided variable `var.bucket_name`.
+
+- **S3 Bucket Server-Side Encryption Configuration**: This resource configures server-side encryption for the S3 bucket, specifying the custom-created KMS key as the master key for encrypting objects stored in the bucket.
+
+- **IAM Policy**: An IAM policy is created to enable specific access to the S3 bucket. This policy is defined via a detailed policy document which grants permissions to perform various actions essential for managing the file carving process. Actions include object retrieval (`GetObject*`), object creation (`PutObject*`), listing the bucket (`ListBucket*`), and managing multipart uploads. It also allows for certain KMS actions necessary for encrypting and decrypting the stored data.
+
+- **IAM Role**: An IAM role (`aws_iam_role`) is provisioned with a trust relationship policy that permits an external entity, specified by `var.fleet_iam_role_arn`, to assume the role. This allows secure access to the S3 bucket and KMS key based on assuming roles across AWS accounts or services.
+
+- **IAM Role Policy Attachment**: This attachment links the previously created IAM policy to the IAM role, ensuring that the permissions are in effect when the role is assumed by the external entity.
+
+## Usage
+
+To use this Terraform configuration, ensure that you have Terraform installed and configured with the necessary AWS credentials. You should define the `bucket_name` and `fleet_iam_role_arn` variables according to your organization's requirements before applying the Terraform plan.
+
+This infrastructure enables secure storage and access for file carving results, facilitating forensic analysis and the capability to respond to security incidents effectively.
+
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_iam_policy.s3_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_role.carve_s3_delegation_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.s3_access_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_kms_key.s3_encryption_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
+| [aws_s3_bucket.carve_results_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [aws_s3_bucket_server_side_encryption_configuration.sse](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.kms_key_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.s3_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_bucket_name"></a> [bucket\_name](#input\_bucket\_name) | The name of the osquery carve results bucket | `string` | n/a | yes |
+| <a name="input_fleet_iam_role_arn"></a> [fleet\_iam\_role\_arn](#input\_fleet\_iam\_role\_arn) | The IAM role ARN of the Fleet service | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_iam_role_arn"></a> [iam\_role\_arn](#output\_iam\_role\_arn) | n/a |
+| <a name="output_s3_bucket_name"></a> [s3\_bucket\_name](#output\_s3\_bucket\_name) | n/a |
+| <a name="output_s3_bucket_region"></a> [s3\_bucket\_region](#output\_s3\_bucket\_region) | n/a |

terraform/addons/byo-file-carving/target-account/outputs.tf

@@ -0,0 +1,11 @@
+output "iam_role_arn" {
+  value = aws_iam_role.carve_s3_delegation_role.arn
+}
+
+output "s3_bucket_name" {
+  value = aws_s3_bucket.carve_results_bucket.id
+}
+
+output "s3_bucket_region" {
+  value = aws_s3_bucket.carve_results_bucket.region
+}

terraform/addons/byo-file-carving/target-account/s3.tf

@@ -0,0 +1,104 @@
+data "aws_caller_identity" "current" {}
+
+# IAM policy document for the KMS key
+data "aws_iam_policy_document" "kms_key_policy" {
+  statement {
+    sid       = "Enable IAM User Permissions"
+    actions   = ["kms:*"]
+    resources = ["*"]
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+  }
+}
+
+# Create a KMS key for encrypting the S3 bucket
+resource "aws_kms_key" "s3_encryption_key" {
+  description = "KMS key for S3 bucket encryption"
+  is_enabled  = true
+  policy      = data.aws_iam_policy_document.kms_key_policy.json
+}
+
+# Create an S3 bucket with server-side encryption using the customer-managed key
+resource "aws_s3_bucket" "carve_results_bucket" {
+  bucket = var.bucket_name
+}
+
+resource "aws_s3_bucket_public_access_block" "carve_results" {
+  bucket = aws_s3_bucket.carve_results_bucket.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "sse" {
+  bucket = aws_s3_bucket.carve_results_bucket.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm     = "aws:kms"
+      kms_master_key_id = aws_kms_key.s3_encryption_key.key_id
+    }
+  }
+}
+
+# Create an IAM policy which allows the necessary S3 actions
+resource "aws_iam_policy" "s3_access_policy" {
+  name   = "s3_access_policy"
+  policy = data.aws_iam_policy_document.s3_policy.json
+}
+
+# IAM policy document
+data "aws_iam_policy_document" "s3_policy" {
+  statement {
+    actions = [
+      "s3:GetObject*",
+      "s3:PutObject*",
+      "s3:ListBucket*",
+      "s3:ListMultipartUploadParts*",
+      "s3:DeleteObject",
+      "s3:CreateMultipartUpload",
+      "s3:AbortMultipartUpload",
+      "s3:ListMultipartUploadParts",
+      "s3:GetBucketLocation"
+    ]
+
+    resources = [
+      aws_s3_bucket.carve_results_bucket.arn,
+      "${aws_s3_bucket.carve_results_bucket.arn}/*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey",
+      "kms:Encrypt"
+    ]
+    resources = [aws_kms_key.s3_encryption_key.arn]
+  }
+}
+
+data "aws_iam_policy_document" "assume_role" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+    principals {
+      identifiers = [var.fleet_iam_role_arn]
+      type        = "AWS"
+    }
+  }
+}
+
+resource "aws_iam_role" "carve_s3_delegation_role" {
+  assume_role_policy = data.aws_iam_policy_document.assume_role.json
+}
+
+# Attach the policy to the role
+resource "aws_iam_role_policy_attachment" "s3_access_attachment" {
+  role       = aws_iam_role.carve_s3_delegation_role.name
+  policy_arn = aws_iam_policy.s3_access_policy.arn
+}

terraform/addons/byo-file-carving/target-account/variables.tf

@@ -0,0 +1,9 @@
+variable "bucket_name" {
+  type = string
+  description = "The name of the osquery carve results bucket"
+}
+
+variable "fleet_iam_role_arn" {
+  type = string
+  description = "The IAM role ARN of the Fleet service"
+}