mirror of
https://github.com/empayre/fleet.git
synced 2024-11-06 17:05:18 +00:00
cross-account firehose destinations via module (#9528)
initial support for cross-account firehose destinations
This commit is contained in:
Benjamin Edwards
GitHub
parent
74617c03af
commit
2f0f549e45
@ -0,0 +1,6 @@
|
||||
# Logging Destination: Firehose
|
||||
This addon provides a Kinesis Firehose logging destination for Fleet.
|
||||
|
||||
First apply the `target-account` module which will provision the necessary bucket, KMS key, and policies.
|
||||
|
||||
Then apply the `firehose` module with the required variables.
|
||||
@ -0,0 +1 @@
|
||||
header-from: .header.md
|
||||
@ -0,0 +1,51 @@
|
||||
# Logging Destination: Firehose
|
||||
This addon provides a Kinesis Firehose logging destination for Fleet with support for cross account S3 delivery.
|
||||
|
||||
## Requirements
|
||||
|
||||
Apply module `target-account` to provision destination bucket, kms key, and IAM policies.
|
||||
|
||||
## Providers
|
||||
|
||||
| Name | Version |
|
||||
|---------------------------------------------------|---------|
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.49.0 |
|
||||
|
||||
## Modules
|
||||
|
||||
No modules.
|
||||
|
||||
## Resources
|
||||
|
||||
| Name | Type |
|
||||
|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|
|
||||
| [aws_iam_policy.firehose-results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
|
||||
| [aws_iam_policy.firehose-status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
|
||||
| [aws_iam_role.firehose-results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
|
||||
| [aws_iam_role.firehose-status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
|
||||
| [aws_iam_role_policy_attachment.firehose-results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
|
||||
| [aws_iam_role_policy_attachment.firehose-status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
|
||||
| [aws_kinesis_firehose_delivery_stream.osquery_results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
|
||||
| [aws_kinesis_firehose_delivery_stream.osquery_status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
|
||||
| [aws_iam_policy_document.osquery_firehose_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
|
||||
| [aws_iam_policy_document.osquery_results_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
|
||||
| [aws_iam_policy_document.osquery_status_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
|
||||
| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
|
||||
|
||||
## Inputs
|
||||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|-------------------------------|----------------------------------------|----------|---------------------|:--------:|
|
||||
| firehose_results_name | n/a | `string` | no default provided | yes |
|
||||
| firehose_status_name | n/a | `string` | no default provided | yes |
|
||||
| customer_prefix | used for resource tagging | `string` | no default provided | yes |
|
||||
| kms_key_arn | key arn used to encrypt target buckets | `string` | no default provided | yes |
|
||||
| results_destination_s3_bucket | bucket name to send osquery results | `string` | no default provided | yes |
|
||||
| status_destination_s3_bucket | bucket name to send osquery status | `string` | no default provided | yes |
|
||||
|
||||
|
||||
## Outputs
|
||||
|
||||
| Name | Description |
|
||||
|-----------------------------------------------------------------------------------------------------------------|-------------|
|
||||
| <a name="output_fleet-extra-env-variables"></a> [fleet-extra-env-variables](#output\_fleet-extra-env-variables) | n/a |
|
||||
@ -0,0 +1,104 @@
|
||||
data "aws_region" "current" {}
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
data "aws_iam_policy_document" "osquery_firehose_assume_role" {
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = ["sts:AssumeRole"]
|
||||
principals {
|
||||
identifiers = ["firehose.amazonaws.com"]
|
||||
type = "Service"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "firehose_policy" {
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = [
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject",
|
||||
"s3:PutObjectAcl" // required according to https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-s3
|
||||
]
|
||||
resources = [
|
||||
"arn:aws:s3:::${var.results_destination_s3_bucket}",
|
||||
"arn:aws:s3:::${var.results_destination_s3_bucket}/*",
|
||||
"arn:aws:s3:::${var.status_destination_s3_bucket}",
|
||||
"arn:aws:s3:::${var.status_destination_s3_bucket}/*"
|
||||
]
|
||||
}
|
||||
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = ["kms:GenerateDataKey*"]
|
||||
resources = [var.kms_key_arn]
|
||||
}
|
||||
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = ["logs:PutLogEvents"]
|
||||
resources = [
|
||||
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_results_name}:*",
|
||||
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_status_name}:*"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "firehose" {
|
||||
name = "${var.customer_prefix}-firehose-cross-account-role"
|
||||
assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "firehose" {
|
||||
policy = data.aws_iam_policy_document.firehose_policy.json
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "firehose" {
|
||||
policy_arn = aws_iam_policy.firehose.arn
|
||||
role = aws_iam_role.firehose.name
|
||||
}
|
||||
|
||||
resource "aws_kinesis_firehose_delivery_stream" "osquery_results" {
|
||||
name = var.firehose_results_name
|
||||
destination = "s3"
|
||||
|
||||
s3_configuration {
|
||||
prefix = var.results_object_prefix
|
||||
role_arn = aws_iam_role.firehose.arn
|
||||
bucket_arn = "arn:aws:s3:::${var.results_destination_s3_bucket}"
|
||||
kms_key_arn = var.kms_key_arn
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_kinesis_firehose_delivery_stream" "osquery_status" {
|
||||
name = var.firehose_status_name
|
||||
destination = "s3"
|
||||
|
||||
s3_configuration {
|
||||
prefix = var.status_object_prefix
|
||||
role_arn = aws_iam_role.firehose
|
||||
bucket_arn = "arn:aws:s3:::${var.status_destination_s3_bucket}"
|
||||
kms_key_arn = var.kms_key_arn
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "firehose-logging" {
|
||||
statement {
|
||||
actions = [
|
||||
"firehose:DescribeDeliveryStream",
|
||||
"firehose:PutRecord",
|
||||
"firehose:PutRecordBatch",
|
||||
]
|
||||
resources = [aws_kinesis_firehose_delivery_stream.osquery_results.arn, aws_kinesis_firehose_delivery_stream.osquery_status.arn]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "firehose-logging" {
|
||||
name = "fleet-firehose-logging"
|
||||
description = "An IAM policy for fleet to log to Firehose destinations"
|
||||
policy = data.aws_iam_policy_document.firehose-logging.json
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
output "fleet_extra_environment_variables" {
|
||||
value = {
|
||||
FLEET_FIREHOSE_STATUS_STREAM = aws_kinesis_firehose_delivery_stream.osquery_status.name
|
||||
FLEET_FIREHOSE_RESULT_STREAM = aws_kinesis_firehose_delivery_stream.osquery_results.name
|
||||
FLEET_FIREHOSE_REGION = data.aws_region.current.name
|
||||
FLEET_OSQUERY_STATUS_LOG_PLUGIN = "firehose"
|
||||
FLEET_OSQUERY_RESULT_LOG_PLUGIN = "firehose"
|
||||
}
|
||||
}
|
||||
|
||||
output "fleet_extra_iam_policies" {
|
||||
value = [
|
||||
aws_iam_policy.firehose-logging.arn
|
||||
]
|
||||
}
|
||||
|
||||
output "firehose_role_arn" {
|
||||
value = aws_iam_role.firehose.arn
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
variable "results_destination_s3_bucket" {
|
||||
type = string
|
||||
description = "s3 bucket name for osquery results"
|
||||
}
|
||||
|
||||
variable "status_destination_s3_bucket" {
|
||||
type = string
|
||||
description = "s3 bucket name for osquery status"
|
||||
}
|
||||
|
||||
variable "kms_key_arn" {
|
||||
type = string
|
||||
description = "kms key arn used to encrypt destination buckets"
|
||||
default = "arn:aws:kms:us-east-2:123456789123:key/fix-me"
|
||||
}
|
||||
|
||||
variable "firehose_results_name" {
|
||||
type = string
|
||||
description = "name of the firehose delivery stream for osquery results logs"
|
||||
}
|
||||
|
||||
variable "firehose_status_name" {
|
||||
type = string
|
||||
description = "name of the firehose delivery stream for osquery status logs"
|
||||
}
|
||||
|
||||
variable "customer_prefix" {
|
||||
type = string
|
||||
description = "customer prefix to use to namespace all resources"
|
||||
}
|
||||
|
||||
variable "results_object_prefix" {
|
||||
type = string
|
||||
description = "object prefix for results logs e.g. 'results/'"
|
||||
default = "results/"
|
||||
}
|
||||
|
||||
variable "status_object_prefix" {
|
||||
type = string
|
||||
description = "object prefix for results logs e.g. 'status/'"
|
||||
default = "status/"
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
terraform {
|
||||
required_version = ">= 1.3.7"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 4.52.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
# Logging Destination: S3
|
||||
This module will provision necessary resources to feed osquery results/status logs into S3.
|
||||
|
||||
## Requirements
|
||||
|
||||
None
|
||||
|
||||
## Providers
|
||||
|
||||
| Name | Version |
|
||||
|---------------------------------------------------|---------|
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.52.0 |
|
||||
|
||||
## Modules
|
||||
|
||||
No modules.
|
||||
|
||||
## Resources
|
||||
|
||||
| Name | Type |
|
||||
|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|
|
||||
| [aws_s3_bucket.osquery-results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
|
||||
| [aws_s3_bucket.osquery-status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
|
||||
| [aws_s3_bucket_acl.osquery-results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource |
|
||||
| [aws_s3_bucket_acl.osquery-status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource |
|
||||
| [aws_s3_bucket_public_access_block.osquery-results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
|
||||
| [aws_s3_bucket_public_access_block.osquery-status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
|
||||
| [aws_s3_bucket_server_side_encryption_configuration.osquery-results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
|
||||
| [aws_s3_bucket_server_side_encryption_configuration.osquery-status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
|
||||
| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
|
||||
|
||||
## Inputs
|
||||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|------------------------|----------------------------------------|----------|---------------------|:--------:|
|
||||
| osquery_results_bucket | name of the bucket for results logging | `string` | no default provided | yes |
|
||||
| osquery_status_bucket | name of the bucket for status logging | `string` | no default provided | yes |
|
||||
| fleet_iam_role_arn | the role ARN from Fleet Cloud | `string` | no default provided | yes |
|
||||
|
||||
## Outputs
|
||||
|
||||
| Name | Description |
|
||||
|---------------------|-------------|
|
||||
| kms_key_arn | n/a |
|
||||
| results_bucket_name | n/a |
|
||||
| status_bucket_name | n/a |
|
||||
@ -0,0 +1,179 @@
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
data "aws_iam_policy_document" "results" {
|
||||
statement {
|
||||
principals {
|
||||
identifiers = [var.fleet_iam_role_arn]
|
||||
type = "AWS"
|
||||
}
|
||||
effect = "Allow"
|
||||
actions = [
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject",
|
||||
"s3:PutObjectAcl" // required according to https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-s3
|
||||
]
|
||||
resources = [
|
||||
aws_s3_bucket.osquery-results.arn,
|
||||
"${aws_s3_bucket.osquery-results.arn}/*"
|
||||
]
|
||||
}
|
||||
|
||||
statement {
|
||||
principals {
|
||||
identifiers = [var.fleet_iam_role_arn]
|
||||
type = "AWS"
|
||||
}
|
||||
effect = "Allow"
|
||||
actions = ["s3:PutObject"]
|
||||
resources = ["${aws_s3_bucket.osquery-results.arn}/*"]
|
||||
condition {
|
||||
test = "StringEquals"
|
||||
values = ["bucket-owner-full-control"]
|
||||
variable = "s3:x-amz-acl"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "status" {
|
||||
statement {
|
||||
principals {
|
||||
identifiers = [var.fleet_iam_role_arn]
|
||||
type = "AWS"
|
||||
}
|
||||
effect = "Allow"
|
||||
actions = [
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject",
|
||||
"s3:PutObjectAcl" // required according to https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-s3
|
||||
]
|
||||
resources = [
|
||||
aws_s3_bucket.osquery-status.arn,
|
||||
"${aws_s3_bucket.osquery-status.arn}/*"
|
||||
]
|
||||
}
|
||||
|
||||
statement {
|
||||
principals {
|
||||
identifiers = [var.fleet_iam_role_arn]
|
||||
type = "AWS"
|
||||
}
|
||||
effect = "Allow"
|
||||
actions = ["s3:PutObject"]
|
||||
resources = ["${aws_s3_bucket.osquery-status.arn}/*"]
|
||||
condition {
|
||||
test = "StringEquals"
|
||||
values = ["bucket-owner-full-control"]
|
||||
variable = "s3:x-amz-acl"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket" "osquery-results" {
|
||||
bucket = var.osquery_results_bucket
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket" "osquery-status" {
|
||||
bucket = var.osquery_status_bucket
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_policy" "results" {
|
||||
bucket = aws_s3_bucket.osquery-results.id
|
||||
policy = data.aws_iam_policy_document.results.json
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_policy" "status" {
|
||||
bucket = aws_s3_bucket.osquery-status.id
|
||||
policy = data.aws_iam_policy_document.status.json
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_public_access_block" "results" {
|
||||
bucket = aws_s3_bucket.osquery-results.id
|
||||
block_public_acls = true
|
||||
block_public_policy = true
|
||||
ignore_public_acls = true
|
||||
restrict_public_buckets = true
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_public_access_block" "status" {
|
||||
bucket = aws_s3_bucket.osquery-status.id
|
||||
block_public_acls = true
|
||||
block_public_policy = true
|
||||
ignore_public_acls = true
|
||||
restrict_public_buckets = true
|
||||
}
|
||||
|
||||
|
||||
resource "aws_s3_bucket_acl" "results" {
|
||||
bucket = aws_s3_bucket.osquery-results.id
|
||||
acl = "private"
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_acl" "status" {
|
||||
bucket = aws_s3_bucket.osquery-status.id
|
||||
acl = "private"
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "key_policy" {
|
||||
// self account has access to key
|
||||
statement {
|
||||
principals {
|
||||
identifiers = [
|
||||
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
|
||||
]
|
||||
type = "AWS"
|
||||
}
|
||||
effect = "Allow"
|
||||
actions = ["*"]
|
||||
resources = ["*"]
|
||||
}
|
||||
|
||||
// only allow the IAM role from fleet aws account
|
||||
statement {
|
||||
principals {
|
||||
identifiers = [var.fleet_iam_role_arn]
|
||||
type = "AWS"
|
||||
}
|
||||
effect = "Allow"
|
||||
actions = ["kms:GenerateDataKey*"]
|
||||
resources = ["*"] // this is basically "self" aka this particular key
|
||||
}
|
||||
}
|
||||
|
||||
// customer managed key to allow other aws account access
|
||||
resource "aws_kms_key" "key" {
|
||||
enable_key_rotation = true
|
||||
policy = data.aws_iam_policy_document.key_policy.json
|
||||
description = "key used for osquery results and status bucket encryption"
|
||||
}
|
||||
|
||||
// enable server side encryption with KMS key
|
||||
resource "aws_s3_bucket_server_side_encryption_configuration" "results" {
|
||||
bucket = aws_s3_bucket.osquery-results.id
|
||||
rule {
|
||||
bucket_key_enabled = true
|
||||
apply_server_side_encryption_by_default {
|
||||
kms_master_key_id = aws_kms_key.key.id
|
||||
sse_algorithm = "aws:kms"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_server_side_encryption_configuration" "status" {
|
||||
bucket = aws_s3_bucket.osquery-status.id
|
||||
rule {
|
||||
bucket_key_enabled = true
|
||||
apply_server_side_encryption_by_default {
|
||||
kms_master_key_id = aws_kms_key.key.id
|
||||
sse_algorithm = "aws:kms"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
output "kms_key_arn" {
|
||||
value = aws_kms_key.key.arn
|
||||
}
|
||||
|
||||
output "results_bucket_name" {
|
||||
value = aws_s3_bucket.osquery-results.id
|
||||
}
|
||||
|
||||
output "status_bucket_name" {
|
||||
value = aws_s3_bucket.osquery-status.id
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
variable "osquery_results_bucket" {
|
||||
type = string
|
||||
description = "name of the bucket to store osquery results logs"
|
||||
}
|
||||
|
||||
variable "osquery_status_bucket" {
|
||||
type = string
|
||||
description = "name of the bucket to store osquery status logs"
|
||||
}
|
||||
|
||||
variable "fleet_iam_role_arn" {
|
||||
type = string
|
||||
description = "the arn of the fleet role that firehose will assume to write data to your bucket"
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
terraform {
|
||||
required_version = ">= 1.3.7"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 4.52.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user