Separate health checks for MySQL and Redis (#6468)

This required a bit of refactoring of some mocking due to how the code
generation does not handle having the same function in different types.

Makefile

@@ -169,7 +169,7 @@ generate-dev: .prefix
 
 generate-mock: .prefix
 	go install github.com/groob/mockimpl@latest
-	go generate github.com/fleetdm/fleet/v4/server/mock
+	go generate github.com/fleetdm/fleet/v4/server/mock github.com/fleetdm/fleet/v4/server/mock/mockresult
 
 deps: deps-js deps-go
 

changes/separate-health-checks

@@ -0,0 +1 @@
+- Allow separate health checks for MySQL and Redis with `/healthz?check=mysql` and `/healthz?check=redis`.

cmd/fleet/serve.go

@@ -6,6 +6,7 @@ import (
 	"crypto/subtle"
 	"crypto/tls"
 	"database/sql/driver"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -418,14 +419,16 @@ the way that the Fleet server works.
 			{
 				// a list of dependencies which could affect the status of the app if unavailable.
 				deps := map[string]interface{}{
-					"datastore":          ds,
-					"query_result_store": resultStore,
+					"mysql": ds,
+					"redis": resultStore,
 				}
 
 				// convert all dependencies to health.Checker if they implement the healthz methods.
 				for name, dep := range deps {
 					if hc, ok := dep.(health.Checker); ok {
 						healthCheckers[name] = hc
+					} else {
+						initFatal(errors.New(name+" should be a health.Checker"), "initializing health checks")
 					}
 				}
 
@@ -639,7 +642,6 @@ func runCrons(
 	failingPoliciesSet fleet.FailingPolicySet,
 	enrollHostLimiter fleet.EnrollHostLimiter,
 ) {
-
 	ourIdentifier, err := server.GenerateRandomText(64)
 	if err != nil {
 		initFatal(ctxerr.New(ctx, "generating random instance identifier"), "")

docs/Using-Fleet/Monitoring-Fleet.md

@@ -15,8 +15,9 @@
 
 Fleet exposes a basic health check at the `/healthz` endpoint. This is the interface to use for simple monitoring and load-balancer health checks.
 
-The `/healthz` endpoint will return an `HTTP 200` status if the server is running and has healthy connections to MySQL and Redis. If there are any problems, the endpoint will return an `HTTP 500` status.
+The `/healthz` endpoint will return an `HTTP 200` status if the server is running and has healthy connections to MySQL and Redis. If there are any problems, the endpoint will return an `HTTP 500` status. Details about failing checks are logged in the Fleet server logs.
 
+Individual checks can be run by providing the `check` URL parameter (eg. `/healthz?check=mysql` or `/healthz?check=redis`).
 ## Metrics
 
 Fleet exposes server metrics in a format compatible with [Prometheus](https://prometheus.io/). A simple example Prometheus configuration is available in [tools/app/prometheus.yml](https://github.com/fleetdm/fleet/blob/194ad5963b0d55bdf976aa93f3de6cabd590c97a/tools/app/prometheus.yml).

server/fleet/datastore.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"time"
+
+	"github.com/fleetdm/fleet/v4/server/health"
 )
 
 type CarveStore interface {
@@ -23,6 +25,8 @@ type CarveStore interface {
 
 // Datastore combines all the interfaces in the Fleet DAL
 type Datastore interface {
+	health.Checker
+
 	CarveStore
 
 	///////////////////////////////////////////////////////////////////////////////

server/health/health.go

@@ -17,8 +17,28 @@ type Checker interface {
 // Handler responds with either:
 // 200 OK if the server can successfully communicate with it's backends or
 // 500 if any of the backends are reporting an issue.
-func Handler(logger log.Logger, checkers map[string]Checker) http.HandlerFunc {
+func Handler(logger log.Logger, allCheckers map[string]Checker) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		checkers := make(map[string]Checker)
+		checks, ok := r.URL.Query()["check"]
+		if ok {
+			if len(checks) == 0 {
+				http.Error(w, "checks must not be empty", http.StatusBadRequest)
+				return
+			}
+			for _, checkName := range checks {
+				check, ok := allCheckers[checkName]
+				if !ok {
+					http.Error(w, "the provided check is not valid", http.StatusBadRequest)
+					return
+				}
+				checkers[checkName] = check
+			}
+
+		} else {
+			checkers = allCheckers
+		}
+
 		healthy := CheckHealth(logger, checkers)
 		if !healthy {
 			w.WriteHeader(http.StatusInternalServerError)

server/health/health_test.go

@@ -35,32 +35,53 @@ func (c fail) HealthCheck() error {
 
 func TestHealthzHandler(t *testing.T) {
 	logger := log.NewNopLogger()
-	failing := Handler(logger, map[string]Checker{
-		"mock": healthcheckFunc(func() error {
-			return errors.New("health check failed")
-		})})
+	failCheck := healthcheckFunc(func() error {
+		return errors.New("health check failed")
+	})
+	passCheck := healthcheckFunc(func() error {
+		return nil
+	})
 
-	ok := Handler(logger, map[string]Checker{
-		"mock": healthcheckFunc(func() error {
-			return nil
-		})})
+	fail := Handler(logger, map[string]Checker{
+		"mock": failCheck,
+	})
+	pass := Handler(logger, map[string]Checker{
+		"mock": passCheck,
+	})
+	both := Handler(logger, map[string]Checker{
+		"pass": passCheck,
+		"fail": failCheck,
+	})
 
-	var httpTests = []struct {
-		wantHeader int
+	httpTests := []struct {
 		handler    http.Handler
+		path       string
+		wantHeader int
 	}{
-		{200, ok},
-		{500, failing},
+		{pass, "/healthz", http.StatusOK},
+		{fail, "/healthz", http.StatusInternalServerError},
+
+		// Empty check name
+		{pass, "/healthz?check=mock&check=", http.StatusBadRequest},
+		// Bad check name
+		{pass, "/healthz?check=mock&check=bad", http.StatusBadRequest},
+		// Passing and failing checks
+		{both, "/healthz", http.StatusInternalServerError},
+		// Passing and failing checks
+		{both, "/healthz?check=pass&check=fail", http.StatusInternalServerError},
+		// Only run passing
+		{both, "/healthz?check=pass", http.StatusOK},
+		// Only run failing
+		{both, "/healthz?check=fail", http.StatusInternalServerError},
 	}
 	for _, tt := range httpTests {
 		t.Run("", func(t *testing.T) {
 			rr := httptest.NewRecorder()
-			req := httptest.NewRequest("GET", "/healthz", nil)
+			req := httptest.NewRequest("GET", tt.path, nil)
 			tt.handler.ServeHTTP(rr, req)
 			assert.Equal(t, rr.Code, tt.wantHeader)
 		})
 	}
-
 }
 
 type healthcheckFunc func() error

server/mock/datastore.go

@@ -7,7 +7,6 @@ import (
 )
 
 //go:generate mockimpl -o datastore_mock.go "s *DataStore" "fleet.Datastore"
-//go:generate mockimpl -o datastore_query_results.go "s *QueryResultStore" "fleet.QueryResultStore"
 
 var _ fleet.Datastore = (*Store)(nil)
 

server/mock/datastore_mock.go

@@ -12,6 +12,8 @@ import (
 
 var _ fleet.Datastore = (*DataStore)(nil)
 
+type HealthCheckFunc func() error
+
 type NewCarveFunc func(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error)
 
 type UpdateCarveFunc func(ctx context.Context, metadata *fleet.CarveMetadata) error
@@ -421,6 +423,9 @@ type InnoDBStatusFunc func(ctx context.Context) (string, error)
 type ProcessListFunc func(ctx context.Context) ([]fleet.MySQLProcess, error)
 
 type DataStore struct {
+	HealthCheckFunc        HealthCheckFunc
+	HealthCheckFuncInvoked bool
+
 	NewCarveFunc        NewCarveFunc
 	NewCarveFuncInvoked bool
 
@@ -1034,6 +1039,11 @@ type DataStore struct {
 	ProcessListFuncInvoked bool
 }
 
+func (s *DataStore) HealthCheck() error {
+	s.HealthCheckFuncInvoked = true
+	return s.HealthCheckFunc()
+}
+
 func (s *DataStore) NewCarve(ctx context.Context, metadata *fleet.CarveMetadata) (*fleet.CarveMetadata, error) {
 	s.NewCarveFuncInvoked = true
 	return s.NewCarveFunc(ctx, metadata)

server/mock/mockresult/generate.go

@@ -0,0 +1,3 @@
+package mock
+
+//go:generate mockimpl -o datastore_query_results.go "s *QueryResultStore" "fleet.QueryResultStore"

server/service/osquery_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/live_query/live_query_mock"
 	"github.com/fleetdm/fleet/v4/server/logging"
 	"github.com/fleetdm/fleet/v4/server/mock"
+	mockresult "github.com/fleetdm/fleet/v4/server/mock/mockresult"
 	"github.com/fleetdm/fleet/v4/server/ptr"
 	"github.com/fleetdm/fleet/v4/server/pubsub"
 	"github.com/fleetdm/fleet/v4/server/service/async"
@@ -1261,7 +1262,7 @@ func TestNewDistributedQueryCampaign(t *testing.T) {
 	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
 		return &fleet.AppConfig{}, nil
 	}
-	rs := &mock.QueryResultStore{
+	rs := &mockresult.QueryResultStore{
 		HealthCheckFunc: func() error {
 			return nil
 		},
@@ -2053,7 +2054,7 @@ func TestDistributedQueriesReloadsHostIfDetailsAreIn(t *testing.T) {
 
 func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) {
 	ds := new(mock.Store)
-	rs := &mock.QueryResultStore{
+	rs := &mockresult.QueryResultStore{
 		HealthCheckFunc: func() error {
 			return nil
 		},
@@ -2126,7 +2127,7 @@ func TestObserversCanOnlyRunDistributedCampaigns(t *testing.T) {
 
 func TestTeamMaintainerCanRunNewDistributedCampaigns(t *testing.T) {
 	ds := new(mock.Store)
-	rs := &mock.QueryResultStore{
+	rs := &mockresult.QueryResultStore{
 		HealthCheckFunc: func() error {
 			return nil
 		},