mirror of
https://github.com/empayre/fleet.git
synced 2024-11-06 08:55:24 +00:00
Feature/infra updates (#2183)
* complete terraform state migration * split firehose results & status streams * extract more variables, with sane defaults * fix fargate configs
This commit is contained in:
Benjamin Edwards
GitHub
parent
db94cf34af
commit
1cc68eea3f
@ -11,6 +11,24 @@ data "aws_iam_policy_document" "fleet" {
|
||||
resources = [aws_secretsmanager_secret.database_password_secret.arn, data.aws_secretsmanager_secret.license.arn]
|
||||
}
|
||||
|
||||
// useful when there is a static number of mysql cluster members
|
||||
dynamic statement {
|
||||
for_each = module.aurora_mysql.rds_cluster_instance_dbi_resource_ids
|
||||
content {
|
||||
effect = "Allow"
|
||||
actions = ["rds-db:connect"]
|
||||
resources = ["arn:aws:rds-db:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:dbuser:${statement.value}/${var.database_user}"]
|
||||
}
|
||||
}
|
||||
|
||||
// allow access to any database via IAM that has the var.database_user user
|
||||
// useful when you are autoscaling mysql read replicas dynamically
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = ["rds-db:connect"]
|
||||
resources = ["arn:aws:rds-db:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:dbuser:*/${var.database_user}"]
|
||||
}
|
||||
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = [
|
||||
@ -18,7 +36,7 @@ data "aws_iam_policy_document" "fleet" {
|
||||
"firehose:PutRecord",
|
||||
"firehose:PutRecordBatch",
|
||||
]
|
||||
resources = [aws_kinesis_firehose_delivery_stream.osquery_logs.arn]
|
||||
resources = [aws_kinesis_firehose_delivery_stream.osquery_results.arn, aws_kinesis_firehose_delivery_stream.osquery_status.arn]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -123,7 +123,7 @@ resource "aws_ecs_task_definition" "backend" {
|
||||
[
|
||||
{
|
||||
name = "fleet"
|
||||
image = "fleetdm/fleet"
|
||||
image = var.image
|
||||
cpu = 512
|
||||
memory = 4096
|
||||
mountPoints = []
|
||||
@ -162,11 +162,11 @@ resource "aws_ecs_task_definition" "backend" {
|
||||
environment = [
|
||||
{
|
||||
name = "FLEET_MYSQL_USERNAME"
|
||||
value = "fleet"
|
||||
value = var.database_user
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_DATABASE"
|
||||
value = "fleet"
|
||||
value = var.database_name
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_ADDRESS"
|
||||
@ -174,11 +174,11 @@ resource "aws_ecs_task_definition" "backend" {
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_READ_REPLICA_USERNAME"
|
||||
value = "fleet"
|
||||
value = var.database_user
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_READ_REPLICA_DATABASE"
|
||||
value = "fleet"
|
||||
value = var.database_name
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_READ_REPLICA_ADDRESS"
|
||||
@ -190,11 +190,11 @@ resource "aws_ecs_task_definition" "backend" {
|
||||
},
|
||||
{
|
||||
name = "FLEET_FIREHOSE_STATUS_STREAM"
|
||||
value = aws_kinesis_firehose_delivery_stream.osquery_logs.name
|
||||
value = aws_kinesis_firehose_delivery_stream.osquery_status.name
|
||||
},
|
||||
{
|
||||
name = "FLEET_FIREHOSE_RESULT_STREAM"
|
||||
value = aws_kinesis_firehose_delivery_stream.osquery_logs.name
|
||||
value = aws_kinesis_firehose_delivery_stream.osquery_results.name
|
||||
},
|
||||
{
|
||||
name = "FLEET_FIREHOSE_REGION"
|
||||
@ -214,32 +214,33 @@ resource "aws_ecs_task_definition" "backend" {
|
||||
},
|
||||
{
|
||||
name = "FLEET_BETA_SOFTWARE_INVENTORY"
|
||||
value = "1"
|
||||
value = var.software_inventory
|
||||
},
|
||||
{
|
||||
name = "FLEET_VULNERABILITIES_DATABASES_PATH"
|
||||
value = "/home/fleet"
|
||||
value = var.vuln_db_path
|
||||
}
|
||||
]
|
||||
}
|
||||
])
|
||||
}
|
||||
|
||||
|
||||
resource "aws_ecs_task_definition" "migration" {
|
||||
family = "fleet-migrate"
|
||||
network_mode = "awsvpc"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
execution_role_arn = aws_iam_role.main.arn
|
||||
task_role_arn = aws_iam_role.main.arn
|
||||
cpu = 256
|
||||
memory = 512
|
||||
cpu = var.cpu_migrate
|
||||
memory = var.mem_migrate
|
||||
container_definitions = jsonencode(
|
||||
[
|
||||
{
|
||||
name = "fleet-prepare-db"
|
||||
image = "fleetdm/fleet"
|
||||
cpu = 256
|
||||
memory = 512
|
||||
image = var.image
|
||||
cpu = var.cpu_migrate
|
||||
memory = var.mem_migrate
|
||||
mountPoints = []
|
||||
volumesFrom = []
|
||||
essential = true
|
||||
@ -259,7 +260,7 @@ resource "aws_ecs_task_definition" "migration" {
|
||||
awslogs-stream-prefix = "fleet"
|
||||
}
|
||||
},
|
||||
command = ["fleet", "prepare", "db"]
|
||||
command = ["fleet", "prepare", "--no-prompt=true", "db"]
|
||||
secrets = [
|
||||
{
|
||||
name = "FLEET_MYSQL_PASSWORD"
|
||||
@ -269,11 +270,11 @@ resource "aws_ecs_task_definition" "migration" {
|
||||
environment = [
|
||||
{
|
||||
name = "FLEET_MYSQL_USERNAME"
|
||||
value = "fleet"
|
||||
value = var.database_user
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_DATABASE"
|
||||
value = "fleet"
|
||||
value = var.database_name
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_ADDRESS"
|
||||
@ -282,15 +283,15 @@ resource "aws_ecs_task_definition" "migration" {
|
||||
{
|
||||
name = "FLEET_REDIS_ADDRESS"
|
||||
value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379"
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
])
|
||||
}
|
||||
|
||||
resource "aws_appautoscaling_target" "ecs_target" {
|
||||
max_capacity = 5
|
||||
min_capacity = 1
|
||||
max_capacity = var.fleet_max_capacity
|
||||
min_capacity = var.fleet_min_capacity
|
||||
resource_id = "service/${aws_ecs_cluster.fleet.name}/${aws_ecs_service.fleet.name}"
|
||||
scalable_dimension = "ecs:service:DesiredCount"
|
||||
service_namespace = "ecs"
|
||||
@ -307,7 +308,7 @@ resource "aws_appautoscaling_policy" "ecs_policy_memory" {
|
||||
predefined_metric_specification {
|
||||
predefined_metric_type = "ECSServiceAverageMemoryUtilization"
|
||||
}
|
||||
target_value = 80
|
||||
target_value = var.memory_tracking_target_value
|
||||
}
|
||||
}
|
||||
|
||||
@ -323,6 +324,14 @@ resource "aws_appautoscaling_policy" "ecs_policy_cpu" {
|
||||
predefined_metric_type = "ECSServiceAverageCPUUtilization"
|
||||
}
|
||||
|
||||
target_value = 60
|
||||
target_value = var.cpu_tracking_target_value
|
||||
}
|
||||
}
|
||||
|
||||
output "fleet_ecs_cluster_arn" {
|
||||
value = aws_ecs_cluster.fleet.arn
|
||||
}
|
||||
|
||||
output "fleet_ecs_cluster_id" {
|
||||
value = aws_ecs_cluster.fleet.id
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
resource "aws_s3_bucket" "osquery" {
|
||||
bucket = "fleet-osquery-logs-archive"
|
||||
resource "aws_s3_bucket" "osquery-results" {
|
||||
bucket = "fleet-osquery-results-archive"
|
||||
acl = "private"
|
||||
|
||||
lifecycle_rule {
|
||||
@ -18,8 +18,27 @@ resource "aws_s3_bucket" "osquery" {
|
||||
}
|
||||
}
|
||||
|
||||
// allow firehose to write to bucket
|
||||
data "aws_iam_policy_document" "osquery_logs_policy_doc" {
|
||||
resource "aws_s3_bucket" "osquery-status" {
|
||||
bucket = "fleet-osquery-status-archive"
|
||||
acl = "private"
|
||||
|
||||
lifecycle_rule {
|
||||
enabled = true
|
||||
expiration {
|
||||
days = 1
|
||||
}
|
||||
}
|
||||
|
||||
server_side_encryption_configuration {
|
||||
rule {
|
||||
apply_server_side_encryption_by_default {
|
||||
sse_algorithm = "aws:kms"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "osquery_results_policy_doc" {
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = [
|
||||
@ -29,22 +48,50 @@ data "aws_iam_policy_document" "osquery_logs_policy_doc" {
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject"
|
||||
]
|
||||
resources = [aws_s3_bucket.osquery.arn, "${aws_s3_bucket.osquery.arn}/*"]
|
||||
resources = [aws_s3_bucket.osquery-results.arn, "${aws_s3_bucket.osquery-results.arn}/*"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "firehose" {
|
||||
name = "osquery_logs_firehose_policy"
|
||||
policy = data.aws_iam_policy_document.osquery_logs_policy_doc.json
|
||||
data "aws_iam_policy_document" "osquery_status_policy_doc" {
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = [
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject"
|
||||
]
|
||||
resources = [aws_s3_bucket.osquery-status.arn, "${aws_s3_bucket.osquery-status.arn}/*"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "firehose" {
|
||||
resource "aws_iam_policy" "firehose-results" {
|
||||
name = "osquery_results_firehose_policy"
|
||||
policy = data.aws_iam_policy_document.osquery_results_policy_doc.json
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "firehose-status" {
|
||||
name = "osquery_status_firehose_policy"
|
||||
policy = data.aws_iam_policy_document.osquery_status_policy_doc.json
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "firehose-results" {
|
||||
assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "firehose" {
|
||||
policy_arn = aws_iam_policy.firehose.arn
|
||||
role = aws_iam_role.firehose.name
|
||||
resource "aws_iam_role" "firehose-status" {
|
||||
assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "firehose-results" {
|
||||
policy_arn = aws_iam_policy.firehose-results.arn
|
||||
role = aws_iam_role.firehose-results.name
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "firehose-status" {
|
||||
policy_arn = aws_iam_policy.firehose-status.arn
|
||||
role = aws_iam_role.firehose-status.name
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "osquery_firehose_assume_role" {
|
||||
@ -58,12 +105,22 @@ data "aws_iam_policy_document" "osquery_firehose_assume_role" {
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_kinesis_firehose_delivery_stream" "osquery_logs" {
|
||||
name = "osquery_logs"
|
||||
resource "aws_kinesis_firehose_delivery_stream" "osquery_results" {
|
||||
name = "osquery_results"
|
||||
destination = "s3"
|
||||
|
||||
s3_configuration {
|
||||
role_arn = aws_iam_role.firehose.arn
|
||||
bucket_arn = aws_s3_bucket.osquery.arn
|
||||
role_arn = aws_iam_role.firehose-results.arn
|
||||
bucket_arn = aws_s3_bucket.osquery-results.arn
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_kinesis_firehose_delivery_stream" "osquery_status" {
|
||||
name = "osquery_status"
|
||||
destination = "s3"
|
||||
|
||||
s3_configuration {
|
||||
role_arn = aws_iam_role.firehose-status.arn
|
||||
bucket_arn = aws_s3_bucket.osquery-status.arn
|
||||
}
|
||||
}
|
||||
@ -7,6 +7,13 @@ provider "aws" {
|
||||
}
|
||||
|
||||
terraform {
|
||||
// these values are hard-coded to prevent chicken before the egg situations
|
||||
backend "s3" {
|
||||
bucket = "fleet-terraform-remote-state"
|
||||
region = "us-east-2"
|
||||
key = "fleet/"
|
||||
dynamodb_table = "fleet-terraform-state-lock"
|
||||
}
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
@ -17,3 +24,37 @@ terraform {
|
||||
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
resource "aws_s3_bucket" "remote_state" {
|
||||
bucket = "${var.prefix}-terraform-remote-state"
|
||||
acl = "private"
|
||||
versioning {
|
||||
enabled = true
|
||||
}
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
tags = {
|
||||
Name = "S3 Remote Terraform State Store"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_public_access_block" "fleet_terraform_state" {
|
||||
bucket = aws_s3_bucket.remote_state.id
|
||||
block_public_acls = true
|
||||
block_public_policy = true
|
||||
}
|
||||
|
||||
resource "aws_dynamodb_table" "fleet_terraform_state_lock" {
|
||||
name = "fleet-terraform-state-lock"
|
||||
hash_key = "LockID"
|
||||
billing_mode = "PAY_PER_REQUEST"
|
||||
|
||||
attribute {
|
||||
name = "LockID"
|
||||
type = "S"
|
||||
}
|
||||
|
||||
tags = {
|
||||
Name = "DynamoDB Terraform State Lock Table"
|
||||
}
|
||||
}
|
||||
@ -5,3 +5,11 @@ output "nameservers_fleetctl" {
|
||||
output "nameservers_fleetdm" {
|
||||
value = aws_route53_zone.dogfood_fleetdm_com.name_servers
|
||||
}
|
||||
|
||||
output "backend_security_group" {
|
||||
value = aws_security_group.backend.arn
|
||||
}
|
||||
|
||||
output "private_subnets" {
|
||||
value = module.vpc.private_subnet_arns
|
||||
}
|
||||
@ -1,14 +1,14 @@
|
||||
resource "aws_route53_zone" "dogfood_fleetctl_com" {
|
||||
name = "dogfood.fleetctl.com"
|
||||
name = var.domain_fleetctl
|
||||
}
|
||||
|
||||
resource "aws_route53_zone" "dogfood_fleetdm_com" {
|
||||
name = "dogfood.fleetdm.com"
|
||||
name = var.domain_fleetdm
|
||||
}
|
||||
|
||||
resource "aws_route53_record" "dogfood_fleetctl_com" {
|
||||
zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id
|
||||
name = "dogfood.fleetctl.com"
|
||||
name = var.domain_fleetctl
|
||||
type = "A"
|
||||
|
||||
alias {
|
||||
@ -20,7 +20,7 @@ resource "aws_route53_record" "dogfood_fleetctl_com" {
|
||||
|
||||
resource "aws_route53_record" "dogfood_fleetdm_com" {
|
||||
zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id
|
||||
name = "dogfood.fleetdm.com"
|
||||
name = var.domain_fleetdm
|
||||
type = "A"
|
||||
|
||||
alias {
|
||||
@ -31,7 +31,7 @@ resource "aws_route53_record" "dogfood_fleetdm_com" {
|
||||
}
|
||||
|
||||
resource "aws_acm_certificate" "dogfood_fleetctl_com" {
|
||||
domain_name = "dogfood.fleetctl.com"
|
||||
domain_name = var.domain_fleetctl
|
||||
validation_method = "DNS"
|
||||
|
||||
lifecycle {
|
||||
@ -40,7 +40,7 @@ resource "aws_acm_certificate" "dogfood_fleetctl_com" {
|
||||
}
|
||||
|
||||
resource "aws_acm_certificate" "dogfood_fleetdm_com" {
|
||||
domain_name = "dogfood.fleetdm.com"
|
||||
domain_name = var.domain_fleetdm
|
||||
validation_method = "DNS"
|
||||
|
||||
lifecycle {
|
||||
|
||||
@ -1,7 +1,3 @@
|
||||
locals {
|
||||
name = "fleetdm"
|
||||
}
|
||||
|
||||
resource "random_password" "database_password" {
|
||||
length = 16
|
||||
special = false
|
||||
@ -56,6 +52,13 @@ resource "aws_secretsmanager_secret_version" "database_password_secret_version"
|
||||
// }
|
||||
//}
|
||||
|
||||
variable "db_instance_type_writer" {
|
||||
default = "db.t4g.medium"
|
||||
}
|
||||
variable "db_instance_type_reader" {
|
||||
default = "db.t4g.medium"
|
||||
}
|
||||
|
||||
module "aurora_mysql" {
|
||||
source = "terraform-aws-modules/rds-aurora/aws"
|
||||
version = "5.2.0"
|
||||
@ -63,15 +66,15 @@ module "aurora_mysql" {
|
||||
name = "${local.name}-mysql-iam"
|
||||
engine = "aurora-mysql"
|
||||
engine_version = "5.7.mysql_aurora.2.10.0"
|
||||
instance_type = "db.t4g.medium"
|
||||
instance_type_replica = "db.t4g.medium"
|
||||
instance_type = var.db_instance_type_writer
|
||||
instance_type_replica = var.db_instance_type_reader
|
||||
|
||||
iam_database_authentication_enabled = true
|
||||
storage_encrypted = true
|
||||
username = "fleet"
|
||||
username = var.database_user
|
||||
password = random_password.database_password.result
|
||||
create_random_password = false
|
||||
database_name = "fleet"
|
||||
database_name = var.database_name
|
||||
enable_http_endpoint = false
|
||||
#performance_insights_enabled = true
|
||||
|
||||
|
||||
@ -2,10 +2,10 @@ variable "maintenance_window" {
|
||||
default = ""
|
||||
}
|
||||
variable "engine_version" {
|
||||
default = "5.0.6"
|
||||
default = "6.x"
|
||||
}
|
||||
variable "node_type" {
|
||||
default = "cache.t2.micro"
|
||||
default = "cache.t3.micro"
|
||||
}
|
||||
variable "number_cache_clusters" {
|
||||
default = 3
|
||||
@ -13,7 +13,7 @@ variable "number_cache_clusters" {
|
||||
resource "aws_elasticache_replication_group" "default" {
|
||||
availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"]
|
||||
engine = "redis"
|
||||
parameter_group_name = aws_elasticache_parameter_group.default.name
|
||||
parameter_group_name = "default.redis6.x"
|
||||
subnet_group_name = module.vpc.elasticache_subnet_group_name
|
||||
security_group_ids = [aws_security_group.redis.id]
|
||||
replication_group_id = "fleetdm-redis"
|
||||
@ -30,12 +30,6 @@ resource "aws_elasticache_replication_group" "default" {
|
||||
replication_group_description = "fleetdm-redis"
|
||||
}
|
||||
|
||||
resource "aws_elasticache_parameter_group" "default" {
|
||||
name = "fleetdm-redis"
|
||||
family = "redis5.0"
|
||||
description = "for fleet"
|
||||
}
|
||||
|
||||
resource "aws_security_group" "redis" {
|
||||
name = local.security_group_name
|
||||
vpc_id = module.vpc.vpc_id
|
||||
|
||||
@ -1,3 +1,70 @@
|
||||
locals {
|
||||
name = "fleetdm"
|
||||
}
|
||||
|
||||
variable "prefix" {
|
||||
default = "fleet"
|
||||
}
|
||||
|
||||
variable "domain_fleetdm" {
|
||||
default = "dogfood.fleetdm.com"
|
||||
}
|
||||
|
||||
variable "domain_fleetctl" {
|
||||
default = "dogfood.fleetctl.com"
|
||||
}
|
||||
|
||||
variable "database_user" {
|
||||
description = "database user fleet will authenticate and query with"
|
||||
default = "fleet"
|
||||
}
|
||||
|
||||
variable "database_name" {
|
||||
description = "the name of the database fleet will create/use"
|
||||
default = "fleet"
|
||||
}
|
||||
|
||||
variable "image" {
|
||||
description = "the name of the container image to run"
|
||||
default = "fleetdm/fleet"
|
||||
}
|
||||
|
||||
variable "software_inventory" {
|
||||
description = "enable/disable software inventory (default is enabled)"
|
||||
default = "1"
|
||||
}
|
||||
|
||||
variable "vuln_db_path" {
|
||||
description = "the path to save the vuln database"
|
||||
default = "/home/fleet"
|
||||
}
|
||||
|
||||
variable "cpu_migrate" {
|
||||
description = "cpu units for migration task"
|
||||
default = 1024
|
||||
}
|
||||
|
||||
variable "mem_migrate" {
|
||||
description = "memory limit for migration task in MB"
|
||||
default = 2048
|
||||
}
|
||||
|
||||
variable "fleet_max_capacity" {
|
||||
description = "maximum number of fleet containers to run"
|
||||
default = 5
|
||||
}
|
||||
|
||||
variable "fleet_min_capacity" {
|
||||
description = "minimum number of fleet containers to run"
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "memory_tracking_target_value" {
|
||||
description = "target memory utilization for target tracking policy (default 80%)"
|
||||
default = 80
|
||||
}
|
||||
|
||||
variable "cpu_tracking_target_value" {
|
||||
description = "target cpu utilization for target tracking policy (default 60%)"
|
||||
default = 60
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user