add health package to create reusable healthz checks. (#1583)

Moved the healthz handler and exposed the CheckHealth method so the same healthchecks could be
used by TLS, gRPC and any other APIs.

cmd/fleet/serve.go

@@ -18,6 +18,7 @@ import (
 	kitprometheus "github.com/go-kit/kit/metrics/prometheus"
 	"github.com/kolide/fleet/server/config"
 	"github.com/kolide/fleet/server/datastore/mysql"
+	"github.com/kolide/fleet/server/health"
 	"github.com/kolide/fleet/server/kolide"
 	"github.com/kolide/fleet/server/launcher"
 	"github.com/kolide/fleet/server/mail"
@@ -186,15 +187,25 @@ the way that the Fleet server works.
 
 			}
 
-			// a list of dependencies which could affect the status of the app if unavailable
-			healthCheckers := map[string]interface{}{
-				"datastore":          ds,
-				"query_result_store": resultStore,
+			healthCheckers := make(map[string]health.Checker)
+			{
+				// a list of dependencies which could affect the status of the app if unavailable.
+				deps := map[string]interface{}{
+					"datastore":          ds,
+					"query_result_store": resultStore,
+				}
+
+				// convert all dependencies to health.Checker if they implement the healthz methods.
+				for name, dep := range deps {
+					if hc, ok := dep.(health.Checker); ok {
+						healthCheckers[name] = hc
+					}
+				}
 			}
 
 			r := http.NewServeMux()
 
-			r.Handle("/healthz", prometheus.InstrumentHandler("healthz", healthz(httpLogger, healthCheckers)))
+			r.Handle("/healthz", prometheus.InstrumentHandler("healthz", health.Handler(httpLogger, healthCheckers)))
 			r.Handle("/version", prometheus.InstrumentHandler("version", version.Handler()))
 			r.Handle("/assets/", prometheus.InstrumentHandler("static_assets", service.ServeStaticAssets("/assets/")))
 			r.Handle("/metrics", prometheus.InstrumentHandler("metrics", promhttp.Handler()))
@@ -270,32 +281,6 @@ the way that the Fleet server works.
 	return serveCmd
 }
 
-// healthz is an http handler which responds with either
-// 200 OK if the server can successfuly communicate with it's backends or
-// 500 if any of the backends are reporting an issue.
-func healthz(logger kitlog.Logger, deps map[string]interface{}) http.HandlerFunc {
-	type healthChecker interface {
-		HealthCheck() error
-	}
-
-	healthy := true
-	return func(w http.ResponseWriter, r *http.Request) {
-		for name, dep := range deps {
-			if hc, ok := dep.(healthChecker); ok {
-				err := hc.HealthCheck()
-				if err != nil {
-					logger.Log("err", err, "health-checker", name)
-					healthy = false
-				}
-			}
-		}
-
-		if !healthy {
-			w.WriteHeader(http.StatusInternalServerError)
-		}
-	}
-}
-
 // Support for TLS security profiles, we set up the TLS configuation based on
 // value supplied to server_tls_compatibility command line flag. The default
 // profile is 'modern'.

server/health/health.go

@@ -0,0 +1,53 @@
+// Package health adds methods for checking the health of service dependencies.
+package health
+
+import (
+	"net/http"
+
+	"github.com/go-kit/kit/log"
+)
+
+// Checker returns an error indicating if a service is in an unhealthy state.
+// Checkers should be implemented by dependencies which can fail, like a DB or mail service.
+type Checker interface {
+	HealthCheck() error
+}
+
+// Handler returns an http.Handler that checks the status of all the dependencies.
+// Handler responds with either:
+// 200 OK if the server can successfuly communicate with it's backends or
+// 500 if any of the backends are reporting an issue.
+func Handler(logger log.Logger, checkers map[string]Checker) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		healthy := CheckHealth(logger, checkers)
+		if !healthy {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// CheckHealth checks multiple checkers returning false if any of them fail.
+// CheckHealth logs the reason a checker fails.
+func CheckHealth(logger log.Logger, checkers map[string]Checker) bool {
+	healthy := true
+	for name, hc := range checkers {
+		if err := hc.HealthCheck(); err != nil {
+			log.With(logger, "component", "healthz").Log("err", err, "health-checker", name)
+			healthy = false
+			continue
+		}
+	}
+	return healthy
+}
+
+// Nop creates a noop checker. Useful in tests.
+func Nop() Checker {
+	return nop{}
+}
+
+type nop struct{}
+
+func (c nop) HealthCheck() error {
+	return nil
+}

server/health/health_test.go

@@ -1,4 +1,4 @@
-package main
+package health
 
 import (
 	"errors"
@@ -8,15 +8,39 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func TestHealthz(t *testing.T) {
+func TestCheckHealth(t *testing.T) {
+	checkers := map[string]Checker{
+		"fail": fail{},
+		"pass": Nop(),
+	}
+
+	healthy := CheckHealth(log.NewNopLogger(), checkers)
+	require.False(t, healthy)
+
+	checkers = map[string]Checker{
+		"pass": Nop(),
+	}
+	healthy = CheckHealth(log.NewNopLogger(), checkers)
+	require.True(t, healthy)
+}
+
+type fail struct{}
+
+func (c fail) HealthCheck() error {
+	return errors.New("fail")
+}
+
+func TestHealthzHandler(t *testing.T) {
 	logger := log.NewNopLogger()
-	failing := healthz(logger, map[string]interface{}{
+	failing := Handler(logger, map[string]Checker{
 		"mock": healthcheckFunc(func() error {
 			return errors.New("health check failed")
 		})})
-	ok := healthz(logger, map[string]interface{}{
+
+	ok := Handler(logger, map[string]Checker{
 		"mock": healthcheckFunc(func() error {
 			return nil
 		})})