mirror of
https://github.com/empayre/fleet.git
synced 2024-11-06 08:55:24 +00:00
Elastic is now managed through ansible for quicker config changes and more stability (#4243)
* Add elasticsearch being managed through ansible for better managability * Testing * testing * testing * testing * testing * testing * testing * fixup * fixup * Added docker stuff * fixup * fixup * fixup * Remove old code * fixup * fix health checks * Add elastic apm configuration * fixup * fixup * fixup * testing * restart always * fixup * fixup
This commit is contained in:
Zachary Winnerman
GitHub
parent
f4d130dde9
commit
899a643c15
@ -1,101 +1,19 @@
|
||||
#!/bin/bash
|
||||
#yum update -y
|
||||
#yum install -y python3-pip git
|
||||
#pip3 install ansible
|
||||
yum update -y
|
||||
yum install -y python3-pip git
|
||||
pip3 install ansible boto3
|
||||
|
||||
TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`
|
||||
REPO=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_repository`
|
||||
PLAYBOOK_PATH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_playbook_path`
|
||||
PLAYBOOK_FILE=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_playbook_file`
|
||||
BRANCH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_branch`
|
||||
export TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`
|
||||
export REPO=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/ansible_repository`
|
||||
export PLAYBOOK_PATH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/ansible_playbook_path`
|
||||
export PLAYBOOK_FILE=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/ansible_playbook_file`
|
||||
export BRANCH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/ansible_branch`
|
||||
export AWS_REGION=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region`
|
||||
|
||||
#git clone "${REPO}" ansible
|
||||
#cd ansible
|
||||
#git checkout "${BRANCH}"
|
||||
#cd "${PLAYBOOK_PATH}"
|
||||
#ansible-playbook -c local "${PLAYBOOK_FILE}"
|
||||
|
||||
yum install -y docker
|
||||
systemctl start docker.service
|
||||
systemctl enable docker.service
|
||||
curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose
|
||||
chmod +x /usr/bin/docker-compose
|
||||
cat << EOT >docker-compose.yml
|
||||
version: '2.2'
|
||||
services:
|
||||
apm-server:
|
||||
image: docker.elastic.co/apm/apm-server:7.15.2
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
kibana:
|
||||
condition: service_healthy
|
||||
cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"]
|
||||
cap_drop: ["ALL"]
|
||||
ports:
|
||||
- 8200:8200
|
||||
networks:
|
||||
- elastic
|
||||
command: >
|
||||
apm-server -e
|
||||
-E apm-server.rum.enabled=true
|
||||
-E setup.kibana.host=kibana:5601
|
||||
-E setup.template.settings.index.number_of_replicas=0
|
||||
-E apm-server.kibana.enabled=true
|
||||
-E apm-server.kibana.host=kibana:5601
|
||||
-E output.elasticsearch.hosts=["elasticsearch:9200"]
|
||||
healthcheck:
|
||||
interval: 10s
|
||||
retries: 12
|
||||
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/
|
||||
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2
|
||||
environment:
|
||||
- bootstrap.memory_lock=true
|
||||
- cluster.name=docker-cluster
|
||||
- cluster.routing.allocation.disk.threshold_enabled=false
|
||||
- discovery.type=single-node
|
||||
- ES_JAVA_OPTS=-XX:UseAVX=2 -Xms1g -Xmx1g
|
||||
ulimits:
|
||||
memlock:
|
||||
hard: -1
|
||||
soft: -1
|
||||
volumes:
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
ports:
|
||||
- 9200:9200
|
||||
networks:
|
||||
- elastic
|
||||
healthcheck:
|
||||
interval: 20s
|
||||
retries: 10
|
||||
test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
|
||||
|
||||
kibana:
|
||||
image: docker.elastic.co/kibana/kibana:7.15.2
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ELASTICSEARCH_URL: http://elasticsearch:9200
|
||||
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||
ports:
|
||||
- 5601:5601
|
||||
networks:
|
||||
- elastic
|
||||
healthcheck:
|
||||
interval: 10s
|
||||
retries: 20
|
||||
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status
|
||||
|
||||
volumes:
|
||||
esdata:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
elastic:
|
||||
driver: bridge
|
||||
EOT
|
||||
|
||||
docker-compose up -d
|
||||
git clone "${REPO}" ansible
|
||||
cd ansible
|
||||
git checkout "${BRANCH}"
|
||||
cd "${PLAYBOOK_PATH}"
|
||||
ansible-playbook -c local "${PLAYBOOK_FILE}"
|
||||
chown -R ansible:ansible ~ansible/ansible
|
||||
rm -rf ansible
|
||||
|
||||
@ -7,7 +7,7 @@ resource "aws_security_group" "elasticsearch" {
|
||||
}
|
||||
|
||||
resource "aws_security_group_rule" "elasticsearch" {
|
||||
description = "${local.prefix}: allow traffic from public internet"
|
||||
description = "${local.prefix}: allow traffic from vpc"
|
||||
type = "ingress"
|
||||
|
||||
from_port = "9200"
|
||||
@ -18,8 +18,20 @@ resource "aws_security_group_rule" "elasticsearch" {
|
||||
security_group_id = aws_security_group.elasticsearch.id
|
||||
}
|
||||
|
||||
resource "aws_security_group_rule" "ssh" {
|
||||
description = "${local.prefix}: allow traffic from vpc"
|
||||
type = "ingress"
|
||||
|
||||
from_port = "22"
|
||||
to_port = "22"
|
||||
protocol = "tcp"
|
||||
cidr_blocks = ["10.0.0.0/8"]
|
||||
|
||||
security_group_id = aws_security_group.elasticsearch.id
|
||||
}
|
||||
|
||||
resource "aws_security_group_rule" "elasticapm" {
|
||||
description = "${local.prefix}: allow traffic from public internet"
|
||||
description = "${local.prefix}: allow traffic from vpc"
|
||||
type = "ingress"
|
||||
|
||||
from_port = "8200"
|
||||
@ -31,7 +43,7 @@ resource "aws_security_group_rule" "elasticapm" {
|
||||
}
|
||||
|
||||
resource "aws_security_group_rule" "kibana" {
|
||||
description = "${local.prefix}: allow traffic from public internet"
|
||||
description = "${local.prefix}: allow traffic from vpc"
|
||||
type = "ingress"
|
||||
|
||||
from_port = "5601"
|
||||
@ -104,17 +116,26 @@ resource "aws_autoscaling_group" "elasticstack" {
|
||||
|
||||
tag {
|
||||
key = "ansible_branch"
|
||||
value = "zwinnerman-add-loadtest-infra-mine-fixup"
|
||||
value = data.git_repository.tf.branch
|
||||
propagate_at_launch = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_instance_profile" "elasticstack" {
|
||||
name = "elasticstack"
|
||||
role = aws_iam_role.elasticstack.name
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "elasticstack" {
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = ["secretsmanager:GetSecretValue"]
|
||||
resources = ["arn:aws:secretsmanager:us-east-2:917007347864:secret:/fleet/ssh/keys-7iQNe1"]
|
||||
}
|
||||
|
||||
statement {
|
||||
actions = ["ec2:DescribeTags"]
|
||||
resources = ["*"]
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "assume_role_es" {
|
||||
@ -164,12 +185,18 @@ resource "aws_launch_template" "elasticstack" {
|
||||
instance_type = "t3.large"
|
||||
key_name = "zwinnerman"
|
||||
vpc_security_group_ids = [aws_security_group.elasticsearch.id]
|
||||
|
||||
metadata_options {
|
||||
instance_metadata_tags = "enabled"
|
||||
http_endpoint = "enabled"
|
||||
http_tokens = "required"
|
||||
}
|
||||
|
||||
user_data = filebase64("${path.module}/elasticsearch.sh")
|
||||
|
||||
iam_instance_profile {
|
||||
arn = aws_iam_instance_profile.elasticstack.arn
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_alb_listener" "elasticsearch" {
|
||||
@ -190,6 +217,9 @@ resource "aws_lb_target_group" "elasticsearch" {
|
||||
port = 9200
|
||||
protocol = "HTTP"
|
||||
vpc_id = module.vpc.vpc_id
|
||||
health_check {
|
||||
path = "/_cat/health"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_alb_listener" "elasticapm" {
|
||||
@ -230,4 +260,7 @@ resource "aws_lb_target_group" "kibana" {
|
||||
port = 5601
|
||||
protocol = "HTTP"
|
||||
vpc_id = module.vpc.vpc_id
|
||||
health_check {
|
||||
path = "/api/status"
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
---
|
||||
- hosts: localhost
|
||||
roles:
|
||||
- role: ec2
|
||||
- role: ansible
|
||||
- role: docker
|
||||
- role: elasticsearch
|
||||
- role: elasticapm
|
||||
- role: kibana
|
||||
@ -0,0 +1,43 @@
|
||||
---
|
||||
- user:
|
||||
name: ansible
|
||||
groups: wheel
|
||||
- lineinfile:
|
||||
path: /etc/sudoers
|
||||
state: present
|
||||
regexp: '^ansible ALL='
|
||||
line: 'ansible ALL=(ALL) NOPASSWD: ALL'
|
||||
validate: /usr/sbin/visudo -cf %s
|
||||
- pip:
|
||||
name:
|
||||
- ansible
|
||||
- boto3
|
||||
- requests
|
||||
- docker
|
||||
state: latest
|
||||
- debug:
|
||||
msg: "{{ instance_tags }}"
|
||||
- git:
|
||||
dest: /home/ansible/ansible
|
||||
version: "{{ instance_tags['tags']['ansible_branch'] }}"
|
||||
repo: "{{ instance_tags['tags']['ansible_repository'] }}"
|
||||
become: no
|
||||
- template:
|
||||
src: "{{ item }}"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
mode: 0444
|
||||
owner: root
|
||||
group: root
|
||||
with_items:
|
||||
- ansible.service
|
||||
- ansible.timer
|
||||
- template:
|
||||
src: run-ansible
|
||||
dest: /usr/local/bin/run-ansible
|
||||
mode: 0555
|
||||
owner: root
|
||||
group: root
|
||||
- systemd:
|
||||
name: ansible.timer
|
||||
state: started
|
||||
enabled: true
|
||||
@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Runs ansible
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/run-ansible
|
||||
User=ansible
|
||||
@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=Runs ansible
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* *:00:00
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`
|
||||
PLAYBOOK_PATH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/ansible_playbook_path`
|
||||
PLAYBOOK_FILE=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/ansible_playbook_file`
|
||||
BRANCH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/ansible_branch`
|
||||
PATH="${PATH}:/usr/local/bin"
|
||||
cd /home/ansible/ansible/"${PLAYBOOK_PATH}"
|
||||
git checkout "${BRANCH}"
|
||||
git pull
|
||||
ansible-playbook --connection=local -b "${PLAYBOOK_FILE}"
|
||||
@ -0,0 +1,3 @@
|
||||
---
|
||||
- package: name=docker
|
||||
- systemd: name=docker.service enabled=true state=started
|
||||
@ -0,0 +1,6 @@
|
||||
---
|
||||
- amazon.aws.ec2_metadata_facts:
|
||||
- amazon.aws.ec2_tag_info:
|
||||
resource: "{{ ansible_ec2_instance_id }}"
|
||||
region: "{{ ansible_ec2_placement_region }}"
|
||||
register: instance_tags
|
||||
@ -0,0 +1,25 @@
|
||||
---
|
||||
- template:
|
||||
src: apm-server.yml
|
||||
dest: /etc/apm-server.yml
|
||||
mode: 0444
|
||||
owner: root
|
||||
group: root
|
||||
- docker_container:
|
||||
name: elasticapm
|
||||
image: docker.elastic.co/apm/apm-server:7.15.2
|
||||
restart_policy: always
|
||||
capabilities:
|
||||
- CHOWN
|
||||
- DAC_OVERRIDE
|
||||
- SETGID
|
||||
- SETUID
|
||||
cap_drop:
|
||||
- ALL
|
||||
volumes:
|
||||
- "/etc/apm-server.yml:/usr/share/apm-server/apm-server.yml:ro"
|
||||
network_mode: host
|
||||
healthcheck:
|
||||
interval: 10s
|
||||
retries: 12
|
||||
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,19 @@
|
||||
---
|
||||
- docker_container:
|
||||
name: elasticsearch
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2
|
||||
env:
|
||||
bootstrap.memory_lock: "true"
|
||||
cluster.name: docker-cluster
|
||||
cluster.routing.allocation.disk.threshold_enabled: "false"
|
||||
discovery.type: single-node
|
||||
ES_JAVA_OPTS: -XX:UseAVX=2 -Xms1g -Xmx1g
|
||||
ulimits:
|
||||
- "memlock:-1:-1"
|
||||
volumes:
|
||||
- "esdata:/usr/share/elasticsearch/data"
|
||||
network_mode: host
|
||||
healthcheck:
|
||||
interval: 20s
|
||||
retries: 10
|
||||
test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
|
||||
@ -0,0 +1,12 @@
|
||||
---
|
||||
- docker_container:
|
||||
name: kibana
|
||||
image: docker.elastic.co/kibana/kibana:7.15.2
|
||||
env:
|
||||
ELASTICSEARCH_URL: http://localhost:9200
|
||||
ELASTICSEARCH_HOSTS: http://localhost:9200
|
||||
network_mode: host
|
||||
healthcheck:
|
||||
interval: 10s
|
||||
retries: 20
|
||||
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status
|
||||
@ -0,0 +1,15 @@
|
||||
---
|
||||
- package: name=openssh-server
|
||||
- ansible.builtin.systemd: name=sshd.service state=started enabled=yes
|
||||
- ansible.builtin.file:
|
||||
path: /home/ec2-user/.ssh
|
||||
owner: ec2-user
|
||||
group: ec2-user
|
||||
mode: 700
|
||||
state: directory
|
||||
- ansible.builtin.template:
|
||||
src: authorized_keys.j2
|
||||
dest: /home/ec2-user/.ssh/authorized_keys
|
||||
owner: ec2-user
|
||||
group: ec2-user
|
||||
mode: 600
|
||||
@ -0,0 +1 @@
|
||||
{{ lookup("amazon.aws.aws_secret", "/fleet/ssh/keys") }}
|
||||
@ -13,5 +13,6 @@ locals {
|
||||
"ELASTIC_APM_SERVICE_NAME" : "fleet"
|
||||
"ELASTIC_APM_ENVIRONMENT" : "loadtest"
|
||||
"ELASTIC_APM_TRANSACTION_SAMPLE_RATE" : "0.004"
|
||||
"ELASTIC_APM_SERVICE_VERSION" : "${var.tag}-${split(":", data.docker_registry_image.dockerhub.sha256_digest)[1]}"
|
||||
}, var.fleet_config) : { name = k, value = v }]
|
||||
}
|
||||
|
||||
@ -23,6 +23,10 @@ terraform {
|
||||
source = "kreuzwerker/docker"
|
||||
version = "~> 2.16.0"
|
||||
}
|
||||
git = {
|
||||
source = "paultyng/git"
|
||||
version = "~> 0.1.0"
|
||||
}
|
||||
}
|
||||
backend "s3" {
|
||||
bucket = "fleet-loadtesting-tfstate"
|
||||
@ -42,3 +46,9 @@ provider "docker" {
|
||||
password = data.aws_ecr_authorization_token.token.password
|
||||
}
|
||||
}
|
||||
|
||||
provider "git" {}
|
||||
|
||||
data "git_repository" "tf" {
|
||||
path = "${path.module}/../../../"
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user