package errorstore import ( "context" "database/sql" "encoding/json" "fmt" "io" "net/http/httptest" "os" "regexp" "strings" "sync/atomic" "testing" "time" "github.com/fleetdm/fleet/v4/server/datastore/redis/redistest" "github.com/fleetdm/fleet/v4/server/fleet" kitlog "github.com/go-kit/kit/log" pkgErrors "github.com/pkg/errors" //nolint:depguard "github.com/rotisserie/eris" //nolint:depguard "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func alwaysErrors() error { return pkgErrors.New("always errors") } func alwaysCallsAlwaysErrors() error { return alwaysErrors() } func alwaysErisErrors() error { return eris.New("always eris errors") } func alwaysNewError(eh *Handler) error { err := eris.New("always new errors") eh.Store(err) return err } func alwaysNewErrorTwo(eh *Handler) error { err := eris.New("always new errors two") eh.Store(err) return err } func alwaysWrappedErr() error { return eris.Wrap(io.EOF, "always EOF") } func TestHashErr(t *testing.T) { t.Run("without stack trace, same error is same hash", func(t *testing.T) { err1 := alwaysErrors() err2 := alwaysCallsAlwaysErrors() assert.Equal(t, hashError(err1), hashError(err2)) }) t.Run("different location, same error is different hash", func(t *testing.T) { err1 := alwaysErisErrors() err2 := alwaysErisErrors() assert.NotEqual(t, hashError(err1), hashError(err2)) }) t.Run("same error, wrapped, same hash", func(t *testing.T) { eris1 := alwaysErisErrors() w1, w2 := fmt.Errorf("wrap: %w", eris1), pkgErrors.Wrap(eris1, "wrap") h1, h2 := hashError(w1), hashError(w2) assert.Equal(t, h1, h2) }) t.Run("generates json", func(t *testing.T) { var m map[string]interface{} generatedErr := pkgErrors.New("some err") res, jsonBytes, err := hashAndMarshalError(generatedErr) require.NoError(t, err) assert.Equal(t, "mWoqz7iS1IPOZXGhpzHLl_DVQOyemWxCmvkpLz8uEZk=", res) assert.True(t, strings.HasPrefix(jsonBytes, `{ "external": "some err`)) require.NoError(t, json.Unmarshal([]byte(jsonBytes), &m)) generatedErr2 := pkgErrors.New("some other err") res, jsonBytes, err = hashAndMarshalError(generatedErr2) require.NoError(t, err) assert.Equal(t, "8AXruOzQmQLF4H3SrzLxXSwFQgZ8DcbkoF1owo0RhTs=", res) assert.True(t, strings.HasPrefix(jsonBytes, `{ "external": "some other err`)) require.NoError(t, json.Unmarshal([]byte(jsonBytes), &m)) }) } func TestHashErrEris(t *testing.T) { t.Run("Marshal", func(t *testing.T) { wd, err := os.Getwd() require.NoError(t, err) generatedErr := eris.New("some err") res, jsonBytes, err := hashAndMarshalError(generatedErr) require.NoError(t, err) assert.NotEmpty(t, res) assert.Regexp(t, regexp.MustCompile(fmt.Sprintf(`\{ "root": \{ "message": "some err", "stack": \[ "errorstore.TestHashErrEris\.func\d+:%s/errors_test\.go:\d+" \] \} \}`, regexp.QuoteMeta(wd))), jsonBytes) }) t.Run("HashWrapped", func(t *testing.T) { // hashing an eris error that wraps a root error hashes to the same // value if it is from the same location, even if wrapped differently // afterwards. err := alwaysWrappedErr() werr1, werr2 := pkgErrors.Wrap(err, "wrap pkg"), fmt.Errorf("wrap fmt: %w", err) wantHash := hashError(err) h1, h2 := hashError(werr1), hashError(werr2) assert.Equal(t, wantHash, h1) assert.Equal(t, wantHash, h2) }) t.Run("HashNew", func(t *testing.T) { err := alwaysErisErrors() werr := eris.Wrap(err, "wrap eris") werr1, werr2 := pkgErrors.Wrap(err, "wrap pkg"), fmt.Errorf("wrap fmt: %w", err) wantHash := hashError(err) h0, h1, h2 := hashError(werr), hashError(werr1), hashError(werr2) assert.Equal(t, wantHash, h0) assert.Equal(t, wantHash, h1) assert.Equal(t, wantHash, h2) }) t.Run("HashSameRootDifferentLocation", func(t *testing.T) { err1 := alwaysWrappedErr() err2 := func() error { return eris.Wrap(io.EOF, "always EOF") }() err3 := func() error { return eris.Wrap(io.EOF, "always EOF") }() h1, h2, h3 := hashError(err1), hashError(err2), hashError(err3) assert.NotEqual(t, h1, h2) assert.NotEqual(t, h1, h3) assert.NotEqual(t, h2, h3) }) } func TestUnwrapAll(t *testing.T) { root := sql.ErrNoRows werr := pkgErrors.Wrap(root, "pkg wrap") gerr := fmt.Errorf("fmt wrap: %w", werr) eerr := eris.Wrap(gerr, "eris wrap") eerr2 := eris.Wrap(eerr, "eris wrap 2") uw := eris.Cause(eerr2) assert.Equal(t, uw, root) assert.Nil(t, eris.Cause(nil)) } func TestErrorHandler(t *testing.T) { t.Run("works if the error handler is down", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // cancel immediately eh := newTestHandler(ctx, nil, kitlog.NewNopLogger(), time.Minute, nil, nil) doneCh := make(chan struct{}) go func() { eh.Store(pkgErrors.New("test")) close(doneCh) }() // should not even block in the call to Store as there is no handler running ticker := time.NewTicker(1 * time.Second) select { case <-doneCh: case <-ticker.C: t.FailNow() } }) t.Run("works if the error storage is disabled", func(t *testing.T) { eh := newTestHandler(context.Background(), nil, kitlog.NewNopLogger(), -1, nil, nil) doneCh := make(chan struct{}) go func() { eh.Store(pkgErrors.New("test")) close(doneCh) }() // should not even block in the call to Store as there is no handler running ticker := time.NewTicker(1 * time.Second) select { case <-doneCh: case <-ticker.C: t.FailNow() } }) wd, err := os.Getwd() require.NoError(t, err) wd = regexp.QuoteMeta(wd) t.Run("standalone", func(t *testing.T) { pool := redistest.SetupRedis(t, "error:", false, false, false) t.Run("collects errors", func(t *testing.T) { testErrorHandlerCollectsErrors(t, pool, wd) }) t.Run("collects different errors", func(t *testing.T) { testErrorHandlerCollectsDifferentErrors(t, pool, wd) }) }) t.Run("cluster", func(t *testing.T) { pool := redistest.SetupRedis(t, "error:", true, true, false) t.Run("collects errors", func(t *testing.T) { testErrorHandlerCollectsErrors(t, pool, wd) }) t.Run("collects different errors", func(t *testing.T) { testErrorHandlerCollectsDifferentErrors(t, pool, wd) }) }) } func testErrorHandlerCollectsErrors(t *testing.T, pool fleet.RedisPool, wd string) { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() chGo, chDone := make(chan struct{}), make(chan struct{}) var storeCalls int32 = 3 testOnStart := func() { close(chGo) } testOnStore := func(err error) { require.NoError(t, err) if atomic.AddInt32(&storeCalls, -1) == 0 { close(chDone) } } eh := newTestHandler(ctx, pool, kitlog.NewNopLogger(), time.Minute, testOnStart, testOnStore) <-chGo for i := 0; i < 3; i++ { alwaysNewError(eh) } <-chDone errors, err := eh.Flush() require.NoError(t, err) require.Len(t, errors, 1) assert.Regexp(t, regexp.MustCompile(fmt.Sprintf(`\{ "root": \{ "message": "always new errors", "stack": \[ "errorstore\.TestErrorHandler\.func\d\.\d+:%s/errors_test\.go:\d+", "errorstore\.testErrorHandlerCollectsErrors:%[1]s/errors_test\.go:\d+", "errorstore\.alwaysNewError:%s/errors_test\.go:\d+" \] \}`, wd, wd)), errors[0]) // and then errors are gone errors, err = eh.Flush() require.NoError(t, err) assert.Len(t, errors, 0) } func testErrorHandlerCollectsDifferentErrors(t *testing.T, pool fleet.RedisPool, wd string) { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() var storeCalls int32 = 5 chGo, chDone := make(chan struct{}), make(chan struct{}) testOnStart := func() { close(chGo) } testOnStore := func(err error) { require.NoError(t, err) if atomic.AddInt32(&storeCalls, -1) == 0 { close(chDone) } } eh := newTestHandler(ctx, pool, kitlog.NewNopLogger(), time.Minute, testOnStart, testOnStore) <-chGo // those two errors are different because from a different strack trace // (different line) alwaysNewError(eh) alwaysNewError(eh) // while those two are the same, only one gets store for i := 0; i < 2; i++ { alwaysNewError(eh) } alwaysNewErrorTwo(eh) <-chDone errors, err := eh.Flush() require.NoError(t, err) require.Len(t, errors, 4) // order is not guaranteed by scan keys for _, jsonErr := range errors { if strings.Contains(jsonErr, "new errors two") { assert.Regexp(t, regexp.MustCompile(fmt.Sprintf(`\{ "root": \{ "message": "always new errors two", "stack": \[ "errorstore\.TestErrorHandler\.func\d\.\d+:%s/errors_test\.go:\d+", "errorstore\.testErrorHandlerCollectsDifferentErrors:%[1]s/errors_test\.go:\d+", "errorstore\.alwaysNewErrorTwo:%[1]s/errors_test\.go:\d+" \] \}`, wd)), jsonErr) } else { assert.Regexp(t, regexp.MustCompile(fmt.Sprintf(`\{ "root": \{ "message": "always new errors", "stack": \[ "errorstore\.TestErrorHandler\.func\d\.\d+:%s/errors_test\.go:\d+", "errorstore\.testErrorHandlerCollectsDifferentErrors:%[1]s/errors_test\.go:\d+", "errorstore\.alwaysNewError:%[1]s/errors_test\.go:\d+" \] \}`, wd)), jsonErr) } } } func TestHttpHandler(t *testing.T) { pool := redistest.SetupRedis(t, "error:", false, false, false) ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() var storeCalls int32 = 2 chGo, chDone := make(chan struct{}), make(chan struct{}) testOnStart := func() { close(chGo) } testOnStore := func(err error) { require.NoError(t, err) if atomic.AddInt32(&storeCalls, -1) == 0 { close(chDone) } } eh := newTestHandler(ctx, pool, kitlog.NewNopLogger(), time.Minute, testOnStart, testOnStore) <-chGo // store two errors alwaysNewError(eh) alwaysNewErrorTwo(eh) <-chDone req := httptest.NewRequest("GET", "/", nil) res := httptest.NewRecorder() eh.ServeHTTP(res, req) require.Equal(t, res.Code, 200) var errs []struct { Root struct { Message string } Wrap []struct { Message string } } require.NoError(t, json.Unmarshal(res.Body.Bytes(), &errs)) require.Len(t, errs, 2) require.NotEmpty(t, errs[0].Root.Message) require.NotEmpty(t, errs[1].Root.Message) }