Feature/infra updates (#2183)

* complete terraform state migration

* split firehose results & status streams

* extract more variables, with sane defaults

* fix fargate configs

tools/terraform/ecs-iam.tf

@@ -11,6 +11,24 @@ data "aws_iam_policy_document" "fleet" {
     resources = [aws_secretsmanager_secret.database_password_secret.arn, data.aws_secretsmanager_secret.license.arn]
   }
 
+  // useful when there is a static number of mysql cluster members
+  dynamic statement {
+    for_each = module.aurora_mysql.rds_cluster_instance_dbi_resource_ids
+    content {
+      effect = "Allow"
+      actions = ["rds-db:connect"]
+      resources = ["arn:aws:rds-db:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:dbuser:${statement.value}/${var.database_user}"]
+    }
+  }
+
+  // allow access to any database via IAM that has the var.database_user user
+  // useful when you are autoscaling mysql read replicas dynamically
+  statement {
+    effect = "Allow"
+    actions = ["rds-db:connect"]
+    resources = ["arn:aws:rds-db:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:dbuser:*/${var.database_user}"]
+  }
+
   statement {
     effect = "Allow"
     actions = [
@@ -18,7 +36,7 @@ data "aws_iam_policy_document" "fleet" {
       "firehose:PutRecord",
       "firehose:PutRecordBatch",
     ]
-    resources = [aws_kinesis_firehose_delivery_stream.osquery_logs.arn]
+    resources = [aws_kinesis_firehose_delivery_stream.osquery_results.arn, aws_kinesis_firehose_delivery_stream.osquery_status.arn]
   }
 }
 

tools/terraform/ecs.tf

@@ -123,7 +123,7 @@ resource "aws_ecs_task_definition" "backend" {
     [
       {
         name        = "fleet"
-        image       = "fleetdm/fleet"
+        image       = var.image
         cpu         = 512
         memory      = 4096
         mountPoints = []
@@ -162,11 +162,11 @@ resource "aws_ecs_task_definition" "backend" {
         environment = [
           {
             name  = "FLEET_MYSQL_USERNAME"
-            value = "fleet"
+            value = var.database_user
           },
           {
             name  = "FLEET_MYSQL_DATABASE"
-            value = "fleet"
+            value = var.database_name
           },
           {
             name  = "FLEET_MYSQL_ADDRESS"
@@ -174,11 +174,11 @@ resource "aws_ecs_task_definition" "backend" {
           },
           {
             name  = "FLEET_MYSQL_READ_REPLICA_USERNAME"
-            value = "fleet"
+            value = var.database_user
           },
           {
             name  = "FLEET_MYSQL_READ_REPLICA_DATABASE"
-            value = "fleet"
+            value = var.database_name
           },
           {
             name  = "FLEET_MYSQL_READ_REPLICA_ADDRESS"
@@ -190,11 +190,11 @@ resource "aws_ecs_task_definition" "backend" {
           },
           {
             name  = "FLEET_FIREHOSE_STATUS_STREAM"
-            value = aws_kinesis_firehose_delivery_stream.osquery_logs.name
+            value = aws_kinesis_firehose_delivery_stream.osquery_status.name
           },
           {
             name  = "FLEET_FIREHOSE_RESULT_STREAM"
-            value = aws_kinesis_firehose_delivery_stream.osquery_logs.name
+            value = aws_kinesis_firehose_delivery_stream.osquery_results.name
           },
           {
             name  = "FLEET_FIREHOSE_REGION"
@@ -214,32 +214,33 @@ resource "aws_ecs_task_definition" "backend" {
           },
           {
             name  = "FLEET_BETA_SOFTWARE_INVENTORY"
-            value = "1"
+            value = var.software_inventory
           },
           {
             name  = "FLEET_VULNERABILITIES_DATABASES_PATH"
-            value = "/home/fleet"
+            value = var.vuln_db_path
           }
         ]
       }
   ])
 }
 
+
 resource "aws_ecs_task_definition" "migration" {
   family                   = "fleet-migrate"
   network_mode             = "awsvpc"
   requires_compatibilities = ["FARGATE"]
   execution_role_arn       = aws_iam_role.main.arn
   task_role_arn            = aws_iam_role.main.arn
-  cpu                      = 256
+  cpu                      = var.cpu_migrate
-  memory                   = 512
+  memory                   = var.mem_migrate
   container_definitions = jsonencode(
     [
       {
         name        = "fleet-prepare-db"
-        image       = "fleetdm/fleet"
+        image       = var.image
-        cpu         = 256
+        cpu         = var.cpu_migrate
-        memory      = 512
+        memory      = var.mem_migrate
         mountPoints = []
         volumesFrom = []
         essential   = true
@@ -259,7 +260,7 @@ resource "aws_ecs_task_definition" "migration" {
             awslogs-stream-prefix = "fleet"
           }
         },
-        command = ["fleet", "prepare", "db"]
+        command = ["fleet", "prepare", "--no-prompt=true", "db"]
         secrets = [
           {
             name      = "FLEET_MYSQL_PASSWORD"
@@ -269,11 +270,11 @@ resource "aws_ecs_task_definition" "migration" {
         environment = [
           {
             name  = "FLEET_MYSQL_USERNAME"
-            value = "fleet"
+            value = var.database_user
           },
           {
             name  = "FLEET_MYSQL_DATABASE"
-            value = "fleet"
+            value = var.database_name
           },
           {
             name  = "FLEET_MYSQL_ADDRESS"
@@ -282,15 +283,15 @@ resource "aws_ecs_task_definition" "migration" {
           {
             name  = "FLEET_REDIS_ADDRESS"
             value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379"
-          }
+          },
         ]
       }
   ])
 }
 
 resource "aws_appautoscaling_target" "ecs_target" {
-  max_capacity       = 5
+  max_capacity       = var.fleet_max_capacity
-  min_capacity       = 1
+  min_capacity       = var.fleet_min_capacity
   resource_id        = "service/${aws_ecs_cluster.fleet.name}/${aws_ecs_service.fleet.name}"
   scalable_dimension = "ecs:service:DesiredCount"
   service_namespace  = "ecs"
@@ -307,7 +308,7 @@ resource "aws_appautoscaling_policy" "ecs_policy_memory" {
     predefined_metric_specification {
       predefined_metric_type = "ECSServiceAverageMemoryUtilization"
     }
-    target_value = 80
+    target_value = var.memory_tracking_target_value
   }
 }
 
@@ -323,6 +324,14 @@ resource "aws_appautoscaling_policy" "ecs_policy_cpu" {
       predefined_metric_type = "ECSServiceAverageCPUUtilization"
     }
 
-    target_value = 60
+    target_value = var.cpu_tracking_target_value
   }
 }
+
+output "fleet_ecs_cluster_arn" {
+  value = aws_ecs_cluster.fleet.arn
+}
+
+output "fleet_ecs_cluster_id" {
+  value = aws_ecs_cluster.fleet.id
+}

tools/terraform/firehose.tf

@@ -1,5 +1,5 @@
-resource "aws_s3_bucket" "osquery" {
+resource "aws_s3_bucket" "osquery-results" {
-  bucket = "fleet-osquery-logs-archive"
+  bucket = "fleet-osquery-results-archive"
   acl    = "private"
 
   lifecycle_rule {
@@ -18,8 +18,27 @@ resource "aws_s3_bucket" "osquery" {
   }
 }
 
-// allow firehose to write to bucket
+resource "aws_s3_bucket" "osquery-status" {
-data "aws_iam_policy_document" "osquery_logs_policy_doc" {
+  bucket = "fleet-osquery-status-archive"
+  acl    = "private"
+
+  lifecycle_rule {
+    enabled = true
+    expiration {
+      days = 1
+    }
+  }
+
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        sse_algorithm = "aws:kms"
+      }
+    }
+  }
+}
+
+data "aws_iam_policy_document" "osquery_results_policy_doc" {
   statement {
     effect = "Allow"
     actions = [
@@ -29,22 +48,50 @@ data "aws_iam_policy_document" "osquery_logs_policy_doc" {
       "s3:ListBucketMultipartUploads",
       "s3:PutObject"
     ]
-    resources = [aws_s3_bucket.osquery.arn, "${aws_s3_bucket.osquery.arn}/*"]
+    resources = [aws_s3_bucket.osquery-results.arn, "${aws_s3_bucket.osquery-results.arn}/*"]
   }
 }
 
-resource "aws_iam_policy" "firehose" {
+data "aws_iam_policy_document" "osquery_status_policy_doc" {
-  name   = "osquery_logs_firehose_policy"
+  statement {
-  policy = data.aws_iam_policy_document.osquery_logs_policy_doc.json
+    effect = "Allow"
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:GetBucketLocation",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:PutObject"
+    ]
+    resources = [aws_s3_bucket.osquery-status.arn, "${aws_s3_bucket.osquery-status.arn}/*"]
+  }
 }
 
-resource "aws_iam_role" "firehose" {
+resource "aws_iam_policy" "firehose-results" {
+  name   = "osquery_results_firehose_policy"
+  policy = data.aws_iam_policy_document.osquery_results_policy_doc.json
+}
+
+resource "aws_iam_policy" "firehose-status" {
+  name   = "osquery_status_firehose_policy"
+  policy = data.aws_iam_policy_document.osquery_status_policy_doc.json
+}
+
+resource "aws_iam_role" "firehose-results" {
   assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json
 }
 
-resource "aws_iam_role_policy_attachment" "firehose" {
+resource "aws_iam_role" "firehose-status" {
-  policy_arn = aws_iam_policy.firehose.arn
+  assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json
-  role       = aws_iam_role.firehose.name
+}
+
+resource "aws_iam_role_policy_attachment" "firehose-results" {
+  policy_arn = aws_iam_policy.firehose-results.arn
+  role       = aws_iam_role.firehose-results.name
+}
+
+resource "aws_iam_role_policy_attachment" "firehose-status" {
+  policy_arn = aws_iam_policy.firehose-status.arn
+  role       = aws_iam_role.firehose-status.name
 }
 
 data "aws_iam_policy_document" "osquery_firehose_assume_role" {
@@ -58,12 +105,22 @@ data "aws_iam_policy_document" "osquery_firehose_assume_role" {
   }
 }
 
-resource "aws_kinesis_firehose_delivery_stream" "osquery_logs" {
+resource "aws_kinesis_firehose_delivery_stream" "osquery_results" {
-  name        = "osquery_logs"
+  name        = "osquery_results"
   destination = "s3"
 
   s3_configuration {
-    role_arn   = aws_iam_role.firehose.arn
+    role_arn   = aws_iam_role.firehose-results.arn
-    bucket_arn = aws_s3_bucket.osquery.arn
+    bucket_arn = aws_s3_bucket.osquery-results.arn
+  }
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "osquery_status" {
+  name        = "osquery_status"
+  destination = "s3"
+
+  s3_configuration {
+    role_arn   = aws_iam_role.firehose-status.arn
+    bucket_arn = aws_s3_bucket.osquery-status.arn
   }
 }

tools/terraform/main.tf

@@ -7,6 +7,13 @@ provider "aws" {
 }
 
 terraform {
+  // these values are hard-coded to prevent chicken before the egg situations
+  backend "s3" {
+    bucket = "fleet-terraform-remote-state"
+    region = "us-east-2"
+    key = "fleet/"
+    dynamodb_table = "fleet-terraform-state-lock"
+  }
   required_providers {
     aws = {
       source  = "hashicorp/aws"
@@ -17,3 +24,37 @@ terraform {
 
 data "aws_caller_identity" "current" {}
 
+resource "aws_s3_bucket" "remote_state" {
+  bucket = "${var.prefix}-terraform-remote-state"
+  acl    = "private"
+  versioning {
+    enabled = true
+  }
+  lifecycle {
+    prevent_destroy = true
+  }
+  tags = {
+    Name = "S3 Remote Terraform State Store"
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "fleet_terraform_state" {
+  bucket              = aws_s3_bucket.remote_state.id
+  block_public_acls   = true
+  block_public_policy = true
+}
+
+resource "aws_dynamodb_table" "fleet_terraform_state_lock" {
+  name         = "fleet-terraform-state-lock"
+  hash_key     = "LockID"
+  billing_mode = "PAY_PER_REQUEST"
+
+  attribute {
+    name = "LockID"
+    type = "S"
+  }
+
+  tags = {
+    Name = "DynamoDB Terraform State Lock Table"
+  }
+}

tools/terraform/outputs.tf

@@ -5,3 +5,11 @@ output "nameservers_fleetctl" {
 output "nameservers_fleetdm" {
   value = aws_route53_zone.dogfood_fleetdm_com.name_servers
 }
+
+output "backend_security_group" {
+  value = aws_security_group.backend.arn
+}
+
+output "private_subnets" {
+  value = module.vpc.private_subnet_arns
+}

tools/terraform/r53.tf

@@ -1,14 +1,14 @@
 resource "aws_route53_zone" "dogfood_fleetctl_com" {
-  name = "dogfood.fleetctl.com"
+  name = var.domain_fleetctl
 }
 
 resource "aws_route53_zone" "dogfood_fleetdm_com" {
-  name = "dogfood.fleetdm.com"
+  name = var.domain_fleetdm
 }
 
 resource "aws_route53_record" "dogfood_fleetctl_com" {
   zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id
-  name    = "dogfood.fleetctl.com"
+  name    = var.domain_fleetctl
   type    = "A"
 
   alias {
@@ -20,7 +20,7 @@ resource "aws_route53_record" "dogfood_fleetctl_com" {
 
 resource "aws_route53_record" "dogfood_fleetdm_com" {
   zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id
-  name    = "dogfood.fleetdm.com"
+  name    = var.domain_fleetdm
   type    = "A"
 
   alias {
@@ -31,7 +31,7 @@ resource "aws_route53_record" "dogfood_fleetdm_com" {
 }
 
 resource "aws_acm_certificate" "dogfood_fleetctl_com" {
-  domain_name       = "dogfood.fleetctl.com"
+  domain_name       = var.domain_fleetctl
   validation_method = "DNS"
 
   lifecycle {
@@ -40,7 +40,7 @@ resource "aws_acm_certificate" "dogfood_fleetctl_com" {
 }
 
 resource "aws_acm_certificate" "dogfood_fleetdm_com" {
-  domain_name       = "dogfood.fleetdm.com"
+  domain_name       = var.domain_fleetdm
   validation_method = "DNS"
 
   lifecycle {

tools/terraform/rds.tf

@@ -1,7 +1,3 @@
-locals {
-  name = "fleetdm"
-}
-
 resource "random_password" "database_password" {
   length  = 16
   special = false
@@ -56,6 +52,13 @@ resource "aws_secretsmanager_secret_version" "database_password_secret_version"
 //  }
 //}
 
+variable "db_instance_type_writer" {
+  default = "db.t4g.medium"
+}
+variable "db_instance_type_reader" {
+  default = "db.t4g.medium"
+}
+
 module "aurora_mysql" {
   source  = "terraform-aws-modules/rds-aurora/aws"
   version = "5.2.0"
@@ -63,15 +66,15 @@ module "aurora_mysql" {
   name                  = "${local.name}-mysql-iam"
   engine                = "aurora-mysql"
   engine_version        = "5.7.mysql_aurora.2.10.0"
-  instance_type         = "db.t4g.medium"
+  instance_type         = var.db_instance_type_writer
-  instance_type_replica = "db.t4g.medium"
+  instance_type_replica = var.db_instance_type_reader
 
   iam_database_authentication_enabled = true
   storage_encrypted                   = true
-  username                            = "fleet"
+  username                            = var.database_user
   password                            = random_password.database_password.result
   create_random_password              = false
-  database_name                       = "fleet"
+  database_name                       = var.database_name
   enable_http_endpoint                = false
   #performance_insights_enabled        = true
 

tools/terraform/redis.tf

@@ -2,10 +2,10 @@ variable "maintenance_window" {
   default = ""
 }
 variable "engine_version" {
-  default = "5.0.6"
+  default = "6.x"
 }
 variable "node_type" {
-  default = "cache.t2.micro"
+  default = "cache.t3.micro"
 }
 variable "number_cache_clusters" {
   default = 3
@@ -13,7 +13,7 @@ variable "number_cache_clusters" {
 resource "aws_elasticache_replication_group" "default" {
   availability_zones            = ["us-east-2a", "us-east-2b", "us-east-2c"]
   engine                        = "redis"
-  parameter_group_name          = aws_elasticache_parameter_group.default.name
+  parameter_group_name          = "default.redis6.x"
   subnet_group_name             = module.vpc.elasticache_subnet_group_name
   security_group_ids            = [aws_security_group.redis.id]
   replication_group_id          = "fleetdm-redis"
@@ -30,12 +30,6 @@ resource "aws_elasticache_replication_group" "default" {
   replication_group_description = "fleetdm-redis"
 }
 
-resource "aws_elasticache_parameter_group" "default" {
-  name        = "fleetdm-redis"
-  family      = "redis5.0"
-  description = "for fleet"
-}
-
 resource "aws_security_group" "redis" {
   name   = local.security_group_name
   vpc_id = module.vpc.vpc_id

tools/terraform/variables.tf

@@ -1,3 +1,70 @@
+locals {
+  name = "fleetdm"
+}
+
 variable "prefix" {
   default = "fleet"
 }
+
+variable "domain_fleetdm" {
+  default = "dogfood.fleetdm.com"
+}
+
+variable "domain_fleetctl" {
+  default = "dogfood.fleetctl.com"
+}
+
+variable "database_user" {
+  description = "database user fleet will authenticate and query with"
+  default     = "fleet"
+}
+
+variable "database_name" {
+  description = "the name of the database fleet will create/use"
+  default = "fleet"
+}
+
+variable "image" {
+  description = "the name of the container image to run"
+  default     = "fleetdm/fleet"
+}
+
+variable "software_inventory" {
+  description = "enable/disable software inventory (default is enabled)"
+  default     = "1"
+}
+
+variable "vuln_db_path" {
+  description = "the path to save the vuln database"
+  default     = "/home/fleet"
+}
+
+variable "cpu_migrate" {
+  description = "cpu units for migration task"
+  default     = 1024
+}
+
+variable "mem_migrate" {
+  description = "memory limit for migration task in MB"
+  default     = 2048
+}
+
+variable "fleet_max_capacity" {
+  description = "maximum number of fleet containers to run"
+  default     = 5
+}
+
+variable "fleet_min_capacity" {
+  description = "minimum number of fleet containers to run"
+  default     = 1
+}
+
+variable "memory_tracking_target_value" {
+  description = "target memory utilization for target tracking policy (default 80%)"
+  default     = 80
+}
+
+variable "cpu_tracking_target_value" {
+  description = "target cpu utilization for target tracking policy (default 60%)"
+  default     = 60
+}