From 653f882c234b748ea2ab9dd41fc36a1dde44f724 Mon Sep 17 00:00:00 2001
From: Zachary Winnerman <98712682+zwinnerman-fleetdm@users.noreply.github.com>
Date: Wed, 17 Aug 2022 12:26:44 -0400
Subject: [PATCH] Add runbooks for sandbox (#7205)

* Add runbooks for sandbox

* Added a note on athena workgroup
---
 infrastructure/sandbox/readme.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/infrastructure/sandbox/readme.md b/infrastructure/sandbox/readme.md
index 6b0512183..3192ffaa2 100644
--- a/infrastructure/sandbox/readme.md
+++ b/infrastructure/sandbox/readme.md
@@ -31,3 +31,18 @@ for i in $((aws dynamodb scan --table-name sandbox-prod-lifecycle | jq -r '.Item
 ```bash
 for i in $(aws dynamodb scan --table-name sandbox-prod-lifecycle | jq -r '.Items[] | select(.State.S == "provisioned") | .ID.S'); do helm uninstall $i; aws dynamodb delete-item --table-name sandbox-prod-lifecycle --key "{\"ID\": {\"S\": \"${i}\"}}"; done
 ```
+
+### Runbooks
+#### 5xx errors
+If you are seeing 5xx errors, find out what instance its from via the saved query here: https://us-east-2.console.aws.amazon.com/athena/home?region=us-east-2#/query-editor
+Make sure you set the workgroup to sandbox-prod-logs otherwise you won't be able to see the saved query.
+
+You can also see errors via the target groups here: https://us-east-2.console.aws.amazon.com/ec2/v2/home?region=us-east-2#TargetGroups:
+
+#### Fleet Logs
+Fleet logs can be accessed via kubectl. Setup kubectl by following thexe instructions: https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html#create-kubeconfig-automatically
+We do not use eksctl since we use terraform managed resources.
+
+#### Database debugging
+Database debugging is accessed through the rds console: https://us-east-2.console.aws.amazon.com/rds/home?region=us-east-2#database:id=sandbox-prod;is-cluster=true
+Currently only database metrics are available because performance insights is not available for serverless RDS