Run go tests with -race at night (#4090)

* Run go tests with race at night

* Add missing run on step

* Fix boolean logic

* Allow manual run for test-go.yaml

* Add slack notification step

* Try global environment variable for cron schedule

* Fix indentation

* Try number 2

* Try setting cron as usual

* Remove global env

* Only send notification in case of failure when running schedule

* Run with race enable to test

* Add more fixes

* Fix github event variable name

* Set timeouts

* Fix slack notification link

* Re-enable if clause

* Last try on Github Actions

* Re-enable the if clause

.github/workflows/test-go.yaml

@@ -11,6 +11,8 @@ on:
     paths:
       - '**.go'
   workflow_dispatch: # Manual
+  schedule:
+    - cron: '0 4 * * *'
 
 jobs:
   test-go:
@@ -21,6 +23,10 @@ jobs:
         mysql: ["mysql:5.7", "mysql:8"]
     runs-on: ${{ matrix.os }}
 
+    env:
+      RACE_ENABLED: false
+      GO_TEST_TIMEOUT: 10m
+
     steps:
     - name: Install Go
       uses: actions/setup-go@v2
@@ -44,11 +50,39 @@ jobs:
         export PATH=$PATH:~/go/bin
         make generate-go
 
+    - name: Set Go race setting on schedule
+      if: github.event.schedule == '0 4 * * *'
+      run: echo "Running go tests with race enabled"
+      env:
+        RACE_ENABLED: true
+        GO_TEST_TIMEOUT: 1h
+
     - name: Run Go Tests
       run: |
-        NETWORK_TEST=1 REDIS_TEST=1 MYSQL_TEST=1 make test-go
+        NETWORK_TEST=1 REDIS_TEST=1 MYSQL_TEST=1 RACE_ENABLED=$RACE_ENABLED GO_TEST_TIMEOUT=$GO_TEST_TIMEOUT make test-go
 
     - name: Upload to Codecov
       uses: codecov/codecov-action@v2
       with:
         files: coverage.txt
+
+    - name: Slack Notification
+      if: github.event.schedule == '0 4 * * *' && failure()
+      uses: slackapi/slack-github-action@v1.18.0
+      with:
+        payload: |
+          {
+            "text": "${{ job.status }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}",
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Go tests result: ${{ job.status }}\nhttps://github.com/fleetdm/fleet/actions/runs/${{  github.run_id }}\n${{ github.event.pull_request.html_url || github.event.head.html_url }}"
+                }
+              }
+            ]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CORE_ENGINEERING_WEBHOOK_URL }}
+        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

Makefile

@@ -10,6 +10,18 @@ REVSHORT = $(shell git rev-parse --short HEAD)
 USER = $(shell whoami)
 DOCKER_IMAGE_NAME = fleetdm/fleet
 
+ifdef RACE_ENABLED
+RACE_ENABLED_VAR := $(RACE_ENABLED)
+else
+RACE_ENABLED_VAR := false
+endif
+
+ifdef GO_TEST_TIMEOUT
+GO_TEST_TIMEOUT_VAR := $(GO_TEST_TIMEOUT)
+else
+GO_TEST_TIMEOUT_VAR := 10m
+endif
+
 ifneq ($(OS), Windows_NT)
 	# If on macOS, set the shell to bash explicitly
 	ifeq ($(shell uname), Darwin)
@@ -118,7 +130,7 @@ dump-test-schema:
 	go run ./tools/dbutils ./server/datastore/mysql/schema.sql
 
 test-go: dump-test-schema generate-mock
-	go test -tags full,fts5,netgo -parallel 8 -coverprofile=coverage.txt -covermode=atomic ./cmd/... ./ee/... ./orbit/... ./pkg/... ./server/... ./tools/...
+	go test -tags full,fts5,netgo -timeout=${GO_TEST_TIMEOUT_VAR} -race=${RACE_ENABLED_VAR} -parallel 8 -coverprofile=coverage.txt -covermode=atomic ./cmd/... ./ee/... ./orbit/... ./pkg/... ./server/... ./tools/...
 
 analyze-go:
 	go test -tags full,fts5,netgo -race -cover ./...