From 1c311b73bed4ad15ed8181ca1b7994f95dedaf60 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Wed, 13 Mar 2024 10:05:46 -0500 Subject: [PATCH 01/36] Fleet in your calendar configs (#17462) Sub-task for #17230 # Configuration changes App configuration: ```yaml integrations: google_calendar: - email: name@service-account.com private_key: *** domain: fleetdm.com ``` Team configuration: ```yaml integrations: google_calendar: email: name@service-account.com enable_calendar_events: true policies: - name: My policy id: 12 webhook_url: https://example.com/policy-remediation ``` Note: Policy is looked up by name when configuration is set. The policy id is set/updated by the server for internal use. # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- cmd/fleetctl/apply_test.go | 116 ++++++++++- cmd/fleetctl/gitops_test.go | 22 +++ .../expectedGetConfigAppConfigJson.json | 3 +- .../expectedGetConfigAppConfigYaml.yml | 1 + ...ectedGetConfigIncludeServerConfigJson.json | 3 +- ...pectedGetConfigIncludeServerConfigYaml.yml | 1 + .../testdata/expectedGetTeamsJson.json | 6 +- .../testdata/expectedGetTeamsYaml.yml | 4 + .../gitops/global_config_no_paths.yml | 4 + .../testdata/gitops/team_config_no_paths.yml | 8 + .../macosSetupExpectedAppConfigEmpty.yml | 1 + .../macosSetupExpectedAppConfigSet.yml | 1 + .../macosSetupExpectedTeam1And2Empty.yml | 4 + .../macosSetupExpectedTeam1And2Set.yml | 4 + .../testdata/macosSetupExpectedTeam1Empty.yml | 2 + ee/server/service/teams.go | 72 ++++++- server/datastore/mysql/policies.go | 37 ++++ server/datastore/mysql/policies_test.go | 42 ++++ server/datastore/mysql/schema.sql | 2 +- server/fleet/app.go | 3 + server/fleet/datastore.go | 1 + server/fleet/integrations.go | 47 ++++- server/fleet/teams.go | 8 + server/mock/datastore_mock.go | 12 ++ server/service/appconfig.go | 1 + server/service/appconfig_test.go | 4 + server/service/client.go | 64 +++++- server/service/integration_core_test.go | 184 ++++++++++++++++++ server/service/integration_enterprise_test.go | 58 ++++++ .../generated_files/appconfig.txt | 4 + 30 files changed, 706 insertions(+), 13 deletions(-) diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index cd9340a19..f53a78313 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -143,8 +143,19 @@ func TestApplyTeamSpecs(t *testing.T) { } agentOpts := json.RawMessage(`{"config":{"foo":"bar"},"overrides":{"platforms":{"darwin":{"foo":"override"}}}}`) + googleCalEmail := "service-valid@example.com" ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { - return &fleet.AppConfig{AgentOptions: &agentOpts, MDM: fleet.MDM{EnabledAndConfigured: true}}, nil + return &fleet.AppConfig{ + AgentOptions: &agentOpts, + MDM: fleet.MDM{EnabledAndConfigured: true}, + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Email: googleCalEmail, + }, + }, + }, + }, nil } ds.SaveTeamFunc = func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { @@ -439,6 +450,109 @@ spec: HostPercentage: 25, }, *teamsByName["team1"].Config.WebhookSettings.HostStatusWebhook, ) + + // Apply calendar integration + validPolicyID := uint(10) + validPolicyName := "validPolicy" + ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + var policies = make(map[string]*fleet.Policy) + for _, name := range names { + if name != validPolicyName { + return nil, ¬FoundError{} + } + policies[name] = &fleet.Policy{ + PolicyData: fleet.PolicyData{ID: validPolicyID, TeamID: &teamsByName["team1"].ID, Name: validPolicyName}, + } + } + return policies, nil + } + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: `+googleCalEmail+` + enable_calendar_events: true + policies: + - name: `+validPolicyName+` + webhook_url: https://example.com/webhook +`, + ) + require.Equal(t, "[+] applied 1 teams\n", runAppForTest(t, []string{"apply", "-f", filename})) + require.NotNil(t, teamsByName["team1"].Config.Integrations.GoogleCalendar) + assert.Equal( + t, fleet.TeamGoogleCalendarIntegration{ + Email: googleCalEmail, + Enable: true, + Policies: []*fleet.PolicyRef{{Name: validPolicyName, ID: validPolicyID}}, + WebhookURL: "https://example.com/webhook", + }, *teamsByName["team1"].Config.Integrations.GoogleCalendar, + ) + + // Apply calendar integration -- invalid email + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: not_present_globally@example.com + enable_calendar_events: true + policies: + - name: `+validPolicyName+` + webhook_url: https://example.com/webhook +`, + ) + + _, err = runAppNoChecks([]string{"apply", "-f", filename}) + assert.ErrorContains(t, err, "email must match a global Google Calendar integration email") + + // Apply calendar integration -- invalid policy name + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: `+googleCalEmail+` + enable_calendar_events: true + policies: + - name: invalidPolicy + webhook_url: https://example.com/webhook +`, + ) + _, err = runAppNoChecks([]string{"apply", "-f", filename}) + assert.ErrorContains(t, err, "name is invalid") + + // Apply calendar integration -- invalid webhook destination + filename = writeTmpYml( + t, ` +apiVersion: v1 +kind: team +spec: + team: + name: team1 + integrations: + google_calendar: + email: `+googleCalEmail+` + enable_calendar_events: true + policies: + - name: `+validPolicyName+` + webhook_url: bozo +`, + ) + _, err = runAppNoChecks([]string{"apply", "-f", filename}) + assert.ErrorContains(t, err, "invalid URI for request") } func writeTmpYml(t *testing.T, contents string) string { diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 52fdba57e..048c1750a 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -360,6 +360,8 @@ func TestFullGlobalGitOps(t *testing.T) { assert.Len(t, appliedScripts, 1) assert.Len(t, appliedMacProfiles, 1) assert.Len(t, appliedWinProfiles, 1) + require.Len(t, savedAppConfig.Integrations.GoogleCalendar, 1) + assert.Equal(t, "service@example.com", savedAppConfig.Integrations.GoogleCalendar[0].Email) } func TestFullTeamGitOps(t *testing.T) { @@ -389,6 +391,13 @@ func TestFullTeamGitOps(t *testing.T) { EnabledAndConfigured: true, WindowsEnabledAndConfigured: true, }, + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Email: "service@example.com", + }, + }, + }, }, nil } @@ -457,6 +466,12 @@ func TestFullTeamGitOps(t *testing.T) { } return nil, nil } + ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + if slices.Contains(names, "policy1") && slices.Contains(names, "policy2") { + return map[string]*fleet.Policy{"policy1": &policy, "policy2": &policy}, nil + } + return nil, nil + } ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, IDs []uint) ([]uint, error) { policyDeleted = true assert.Equal(t, []uint{policy.ID}, IDs) @@ -536,6 +551,10 @@ func TestFullTeamGitOps(t *testing.T) { assert.Len(t, appliedWinProfiles, 1) assert.True(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) assert.Equal(t, "https://example.com/host_status_webhook", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) + require.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) + assert.Equal(t, "service@example.com", savedTeam.Config.Integrations.GoogleCalendar.Email) + assert.True(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) + assert.Len(t, savedTeam.Config.Integrations.GoogleCalendar.Policies, 2) // Now clear the settings tmpFile, err := os.CreateTemp(t.TempDir(), "*.yml") @@ -569,6 +588,9 @@ team_settings: assert.Equal(t, secret, enrolledSecrets[0].Secret) assert.False(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) assert.Equal(t, "", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) + assert.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) + assert.False(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) + assert.Empty(t, savedTeam.Config.Integrations.GoogleCalendar) assert.Empty(t, savedTeam.Config.MDM.MacOSSettings.CustomSettings) assert.Empty(t, savedTeam.Config.MDM.WindowsSettings.CustomSettings.Value) assert.Empty(t, savedTeam.Config.MDM.MacOSUpdates.Deadline.Value) diff --git a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json index cf01e9134..985bd03f3 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json @@ -79,7 +79,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "mdm": { "apple_bm_terms_expired": false, diff --git a/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml b/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml index 759734585..1e517e952 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml @@ -11,6 +11,7 @@ spec: enable_host_users: true enable_software_inventory: false integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json index 216dbd75b..25550bf50 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json @@ -119,7 +119,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "update_interval": { "osquery_detail": "1h0m0s", diff --git a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml index 3d614097f..4ed7f3ed7 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml @@ -11,6 +11,7 @@ spec: enable_host_users: true enable_software_inventory: false integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/expectedGetTeamsJson.json b/cmd/fleetctl/testdata/expectedGetTeamsJson.json index d19784f2f..08f5fcbf3 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsJson.json +++ b/cmd/fleetctl/testdata/expectedGetTeamsJson.json @@ -22,7 +22,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "features": { "enable_host_users": true, @@ -92,7 +93,8 @@ }, "integrations": { "jira": null, - "zendesk": null + "zendesk": null, + "google_calendar": null }, "features": { "enable_host_users": false, diff --git a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml index 1249b3e5f..c0e3ff6dc 100644 --- a/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetTeamsYaml.yml @@ -9,6 +9,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_updates: @@ -49,6 +51,8 @@ spec: host_expiry_settings: host_expiry_enabled: true host_expiry_window: 15 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_updates: diff --git a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml index 7d0b81e96..cf2eeece6 100644 --- a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml @@ -137,6 +137,10 @@ org_settings: integrations: jira: [] zendesk: [] + google_calendar: + - email: service@example.com + private_key: google_calendar_private_key + domain: example.com mdm: apple_bm_default_team: "" end_user_authentication: diff --git a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml index 3295e75bb..608bafcf7 100644 --- a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml @@ -15,6 +15,14 @@ team_settings: host_expiry_settings: host_expiry_enabled: true host_expiry_window: 30 + integrations: + google_calendar: + email: service@example.com + enable_calendar_events: true + policies: + - name: policy1 + - name: policy2 + webhook_url: https://example.com/google_calendar_webhook agent_options: command_line_flags: distributed_denylist_duration: 0 diff --git a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml index a453a2f7f..cf831eac6 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml @@ -11,6 +11,7 @@ spec: host_expiry_enabled: false host_expiry_window: 0 integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml index 582692272..7fed6f883 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml @@ -11,6 +11,7 @@ spec: host_expiry_enabled: false host_expiry_window: 0 integrations: + google_calendar: null jira: null zendesk: null mdm: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml index 7315325b4..036b0320e 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Empty.yml @@ -9,6 +9,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: @@ -40,6 +42,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml index 1cce56630..d281e9089 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1And2Set.yml @@ -9,6 +9,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: @@ -40,6 +42,8 @@ spec: host_expiry_settings: host_expiry_enabled: false host_expiry_window: 0 + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: diff --git a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml index c6e8b1653..b685d3488 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedTeam1Empty.yml @@ -9,6 +9,8 @@ spec: features: enable_host_users: false enable_software_inventory: false + integrations: + google_calendar: null mdm: enable_disk_encryption: false macos_settings: diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index 95588cd46..f0ad454df 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "strings" "github.com/fleetdm/fleet/v4/server" "github.com/fleetdm/fleet/v4/server/authz" @@ -1068,6 +1070,15 @@ func (svc *Service) editTeamFromSpec( fleet.ValidateEnabledHostStatusIntegrations(*spec.WebhookSettings.HostStatusWebhook, invalid) team.Config.WebhookSettings.HostStatusWebhook = spec.WebhookSettings.HostStatusWebhook } + + if spec.Integrations.GoogleCalendar != nil { + err = svc.validateTeamCalendarIntegrations(ctx, team, spec.Integrations.GoogleCalendar, appCfg, invalid) + if err != nil { + return ctxerr.Wrap(ctx, err, "validate team calendar integrations") + } + team.Config.Integrations.GoogleCalendar = spec.Integrations.GoogleCalendar + } + if invalid.HasErrors() { return ctxerr.Wrap(ctx, invalid) } @@ -1124,7 +1135,9 @@ func (svc *Service) editTeamFromSpec( } if didUpdateMacOSEndUserAuth { - if err := svc.updateMacOSSetupEnableEndUserAuth(ctx, spec.MDM.MacOSSetup.EnableEndUserAuthentication, &team.ID, &team.Name); err != nil { + if err := svc.updateMacOSSetupEnableEndUserAuth( + ctx, spec.MDM.MacOSSetup.EnableEndUserAuthentication, &team.ID, &team.Name, + ); err != nil { return err } } @@ -1132,6 +1145,63 @@ func (svc *Service) editTeamFromSpec( return nil } +func (svc *Service) validateTeamCalendarIntegrations( + ctx context.Context, team *fleet.Team, calendarIntegration *fleet.TeamGoogleCalendarIntegration, + appCfg *fleet.AppConfig, invalid *fleet.InvalidArgumentError, +) error { + if !calendarIntegration.Enable { + return nil + } + // Validate email + emailValid := false + calendarIntegration.Email = strings.TrimSpace(calendarIntegration.Email) + for _, globalCals := range appCfg.Integrations.GoogleCalendar { + if globalCals.Email == calendarIntegration.Email { + emailValid = true + break + } + } + if !emailValid { + invalid.Append("integrations.google_calendar.email", "email must match a global Google Calendar integration email") + } + // Validate URL + if u, err := url.ParseRequestURI(calendarIntegration.WebhookURL); err != nil { + invalid.Append("integrations.google_calendar.webhook_url", err.Error()) + } else if u.Scheme != "https" && u.Scheme != "http" { + invalid.Append("integrations.google_calendar.webhook_url", "webhook_url must be https or http") + } + // Validate policy ids + if len(calendarIntegration.Policies) == 0 { + invalid.Append("integrations.google_calendar.policies", "policies are required") + } + if len(calendarIntegration.Policies) > 0 { + for _, policy := range calendarIntegration.Policies { + policy.Name = strings.TrimSpace(policy.Name) + } + calendarIntegration.Policies = server.RemoveDuplicatesFromSlice(calendarIntegration.Policies) + policyNames := make([]string, 0, len(calendarIntegration.Policies)) + for _, policy := range calendarIntegration.Policies { + policyNames = append(policyNames, policy.Name) + } + // Policies must be team policies. Global policies are not allowed. + policyMap, err := svc.ds.PoliciesByName(ctx, policyNames, team.ID) + if err != nil { + level.Error(svc.logger).Log("msg", "error getting policies by name", "names", policyNames, "err", err) + if fleet.IsNotFound(err) { + invalid.Append("integrations.google_calendar.policies[].name", "name is invalid") + } else { + return err + } + } else { + // PoliciesByName guarantees that all policies are present + for _, policy := range calendarIntegration.Policies { + policy.ID = policyMap[policy.Name].ID + } + } + } + return nil +} + func (svc *Service) applyTeamMacOSSettings(ctx context.Context, spec *fleet.TeamSpec, applyUpon *fleet.MacOSSettings) error { oldCustomSettings := applyUpon.CustomSettings setFields, err := applyUpon.FromMap(spec.MDM.MacOSSettings) diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 69567ec8c..39c7d3e05 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "golang.org/x/text/unicode/norm" "sort" @@ -444,6 +445,42 @@ func (ds *Datastore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fl return policiesByID, nil } +func (ds *Datastore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + sqlQuery := `SELECT ` + policyCols + ` + FROM policies p + WHERE p.team_id = ? AND p.name IN (?)` + query, args, err := sqlx.In(sqlQuery, teamID, names) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "building query to get policies by name") + } + + var policies []*fleet.Policy + err = sqlx.SelectContext( + ctx, + ds.reader(ctx), + &policies, + query, args..., + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(fmt.Sprintf("%v", names))) + } + return nil, ctxerr.Wrap(ctx, err, "getting policies by name") + } + + policiesByName := make(map[string]*fleet.Policy, len(names)) + for _, p := range policies { + policiesByName[p.Name] = p + } + for _, name := range names { + if policiesByName[name] == nil { + return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(name)) + } + } + + return policiesByName, nil +} + func (ds *Datastore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { return deletePolicyDB(ctx, ds.writer(ctx), ids, nil) } diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 5e16c9a40..52da13348 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -38,6 +38,7 @@ func TestPolicies(t *testing.T) { {"PolicyQueriesForHost", testPolicyQueriesForHost}, {"PolicyQueriesForHostPlatforms", testPolicyQueriesForHostPlatforms}, {"PoliciesByID", testPoliciesByID}, + {"PoliciesByName", testPoliciesByName}, {"TeamPolicyTransfer", testTeamPolicyTransfer}, {"ApplyPolicySpec", testApplyPolicySpec}, {"Save", testPoliciesSave}, @@ -1114,6 +1115,47 @@ func testPoliciesByID(t *testing.T, ds *Datastore) { require.ErrorAs(t, err, &nfe) } +func testPoliciesByName(t *testing.T, ds *Datastore) { + ctx := context.Background() + user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) + policyName1 := "policy1" + policyName2 := "policy2" + _ = newTestPolicy(t, ds, user1, policyName1, "darwin", nil) + _ = newTestPolicy(t, ds, user1, policyName2, "darwin", nil) + team1, err := ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team1"}) + require.NoError(t, err) + + // No names provided + _, err = ds.PoliciesByName(context.Background(), []string{}, team1.ID) + require.Error(t, err) + + // Policies don't belong to a team + _, err = ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) + require.Error(t, err) + var nfe fleet.NotFoundError + require.ErrorAs(t, err, &nfe) + + policy1 := newTestPolicy(t, ds, user1, policyName1, "darwin", &team1.ID) + policy2 := newTestPolicy(t, ds, user1, policyName2, "darwin", &team1.ID) + + policiesByName, err := ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) + require.NoError(t, err) + require.Len(t, policiesByName, 2) + assert.Equal(t, policiesByName[policyName1].ID, policy1.ID) + assert.Equal(t, policiesByName[policyName2].ID, policy2.ID) + assert.Equal(t, policiesByName[policyName1].Name, policy1.Name) + assert.Equal(t, policiesByName[policyName2].Name, policy2.Name) + + // Policy does not exist + _, err = ds.PoliciesByName(context.Background(), []string{"doesn't exist"}, team1.ID) + assert.ErrorAs(t, err, &nfe) + + // One exists and one doesn't + _, err = ds.PoliciesByName(context.Background(), []string{policyName1, "doesn't exist"}, team1.ID) + assert.ErrorAs(t, err, &nfe) + +} + func testTeamPolicyTransfer(t *testing.T, ds *Datastore) { ctx := context.Background() user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index a35608e66..0040d4990 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -41,7 +41,7 @@ CREATE TABLE `app_config_json` ( UNIQUE KEY `id` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); +INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `carve_blocks` ( diff --git a/server/fleet/app.go b/server/fleet/app.go index 778f6fe7e..4b936063a 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -488,6 +488,9 @@ func (c *AppConfig) Obfuscate() { for _, zdIntegration := range c.Integrations.Zendesk { zdIntegration.APIToken = MaskedPassword } + for _, calIntegration := range c.Integrations.GoogleCalendar { + calIntegration.PrivateKey = MaskedPassword + } } // Clone implements cloner. diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 4081db8af..69af794e7 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -588,6 +588,7 @@ type Datastore interface { ListGlobalPolicies(ctx context.Context, opts ListOptions) ([]*Policy, error) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*Policy, error) + PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*Policy, error) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) CountPolicies(ctx context.Context, teamID *uint, matchQuery string) (int, error) UpdateHostPolicyCounts(ctx context.Context) error diff --git a/server/fleet/integrations.go b/server/fleet/integrations.go index 131af8880..5f3c5e440 100644 --- a/server/fleet/integrations.go +++ b/server/fleet/integrations.go @@ -13,8 +13,9 @@ import ( // TeamIntegrations contains the configuration for external services' // integrations for a specific team. type TeamIntegrations struct { - Jira []*TeamJiraIntegration `json:"jira"` - Zendesk []*TeamZendeskIntegration `json:"zendesk"` + Jira []*TeamJiraIntegration `json:"jira"` + Zendesk []*TeamZendeskIntegration `json:"zendesk"` + GoogleCalendar *TeamGoogleCalendarIntegration `json:"google_calendar"` } // MatchWithIntegrations matches the team integrations to their corresponding @@ -110,6 +111,17 @@ func (z TeamZendeskIntegration) UniqueKey() string { return z.URL + "\n" + strconv.FormatInt(z.GroupID, 10) } +type TeamGoogleCalendarIntegration struct { + Email string `json:"email"` + Enable bool `json:"enable_calendar_events"` + Policies []*PolicyRef `json:"policies"` + WebhookURL string `json:"webhook_url"` +} +type PolicyRef struct { + Name string `json:"name"` + ID uint `json:"id"` +} + // JiraIntegration configures an instance of an integration with the Jira // system. type JiraIntegration struct { @@ -335,10 +347,17 @@ func makeTestZendeskRequest(ctx context.Context, intg *ZendeskIntegration) error return nil } +type GoogleCalendarIntegration struct { + Email string `json:"email"` + PrivateKey string `json:"private_key"` + Domain string `json:"domain"` +} + // Integrations configures the integrations with external systems. type Integrations struct { - Jira []*JiraIntegration `json:"jira"` - Zendesk []*ZendeskIntegration `json:"zendesk"` + Jira []*JiraIntegration `json:"jira"` + Zendesk []*ZendeskIntegration `json:"zendesk"` + GoogleCalendar []*GoogleCalendarIntegration `json:"google_calendar"` } // ValidateEnabledHostStatusIntegrations checks that the host status integrations @@ -359,6 +378,26 @@ func ValidateEnabledHostStatusIntegrations(webhook HostStatusWebhookSettings, in } } +func ValidateGoogleCalendarIntegrations(intgs []*GoogleCalendarIntegration, invalid *InvalidArgumentError) { + if len(intgs) > 1 { + invalid.Append("integrations.google_calendar", "only one Google Calendar integration is allowed at this time") + } + for _, intg := range intgs { + intg.Email = strings.TrimSpace(intg.Email) + if intg.Email == "" { + invalid.Append("integrations.google_calendar.email", "email is required") + } + intg.PrivateKey = strings.TrimSpace(intg.PrivateKey) + if intg.PrivateKey == "" || intg.PrivateKey == MaskedPassword { + invalid.Append("integrations.google_calendar.private_key", "private_key is required") + } + intg.Domain = strings.TrimSpace(intg.Domain) + if intg.Domain == "" { + invalid.Append("integrations.google_calendar.domain", "domain is required") + } + } +} + // ValidateEnabledVulnerabilitiesIntegrations checks that a single integration // is enabled for vulnerabilities. It adds any error it finds to the invalid // argument error, that can then be checked after the call for errors using diff --git a/server/fleet/teams.go b/server/fleet/teams.go index d1e3a86e9..aee5c7ab9 100644 --- a/server/fleet/teams.go +++ b/server/fleet/teams.go @@ -404,12 +404,20 @@ type TeamSpec struct { MDM TeamSpecMDM `json:"mdm"` Scripts optjson.Slice[string] `json:"scripts"` WebhookSettings TeamSpecWebhookSettings `json:"webhook_settings"` + Integrations TeamSpecIntegrations `json:"integrations"` } type TeamSpecWebhookSettings struct { HostStatusWebhook *HostStatusWebhookSettings `json:"host_status_webhook"` } +// TeamSpecIntegrations contains the configuration for external services' +// integrations for a specific team. +type TeamSpecIntegrations struct { + // If value is nil, we don't want to change the existing value. + GoogleCalendar *TeamGoogleCalendarIntegration `json:"google_calendar"` +} + // TeamSpecFromTeam returns a TeamSpec constructed from the given Team. func TeamSpecFromTeam(t *Team) (*TeamSpec, error) { features, err := json.Marshal(t.Config.Features) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 146982697..603d0616f 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -432,6 +432,8 @@ type ListGlobalPoliciesFunc func(ctx context.Context, opts fleet.ListOptions) ([ type PoliciesByIDFunc func(ctx context.Context, ids []uint) (map[uint]*fleet.Policy, error) +type PoliciesByNameFunc func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) + type DeleteGlobalPoliciesFunc func(ctx context.Context, ids []uint) ([]uint, error) type CountPoliciesFunc func(ctx context.Context, teamID *uint, matchQuery string) (int, error) @@ -1480,6 +1482,9 @@ type DataStore struct { PoliciesByIDFunc PoliciesByIDFunc PoliciesByIDFuncInvoked bool + PoliciesByNameFunc PoliciesByNameFunc + PoliciesByNameFuncInvoked bool + DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFuncInvoked bool @@ -3571,6 +3576,13 @@ func (s *DataStore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fle return s.PoliciesByIDFunc(ctx, ids) } +func (s *DataStore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { + s.mu.Lock() + s.PoliciesByNameFuncInvoked = true + s.mu.Unlock() + return s.PoliciesByNameFunc(ctx, names, teamID) +} + func (s *DataStore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { s.mu.Lock() s.DeleteGlobalPoliciesFuncInvoked = true diff --git a/server/service/appconfig.go b/server/service/appconfig.go index c92cd344c..88ee842e3 100644 --- a/server/service/appconfig.go +++ b/server/service/appconfig.go @@ -385,6 +385,7 @@ func (svc *Service) ModifyAppConfig(ctx context.Context, p []byte, applyOpts fle appConfig.ServerSettings.EnableAnalytics = true } + fleet.ValidateGoogleCalendarIntegrations(appConfig.Integrations.GoogleCalendar, invalid) fleet.ValidateEnabledVulnerabilitiesIntegrations(appConfig.WebhookSettings.VulnerabilitiesWebhook, appConfig.Integrations, invalid) fleet.ValidateEnabledFailingPoliciesIntegrations(appConfig.WebhookSettings.FailingPoliciesWebhook, appConfig.Integrations, invalid) fleet.ValidateEnabledHostStatusIntegrations(appConfig.WebhookSettings.HostStatusWebhook, invalid) diff --git a/server/service/appconfig_test.go b/server/service/appconfig_test.go index 884700bd7..e23f3b18b 100644 --- a/server/service/appconfig_test.go +++ b/server/service/appconfig_test.go @@ -489,6 +489,9 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { Zendesk: []*fleet.ZendeskIntegration{ {APIToken: "zendesktoken"}, }, + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + {PrivateKey: "google-calendar-private-key"}, + }, }, }, nil } @@ -566,6 +569,7 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { require.Equal(t, ac.SMTPSettings.SMTPPassword, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Jira[0].APIToken, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Zendesk[0].APIToken, fleet.MaskedPassword) + require.Equal(t, ac.Integrations.GoogleCalendar[0].PrivateKey, fleet.MaskedPassword) } }) } diff --git a/server/service/client.go b/server/service/client.go index f26bfcc05..b1eb35d69 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -880,14 +880,30 @@ func (c *Client) DoGitOps( } var mdmAppConfig map[string]interface{} var team map[string]interface{} + var teamCalendarIntegration map[string]interface{} if config.TeamName == nil { group.AppConfig = config.OrgSettings group.EnrollSecret = &fleet.EnrollSecretSpec{Secrets: config.OrgSettings["secrets"].([]*fleet.EnrollSecret)} group.AppConfig.(map[string]interface{})["agent_options"] = config.AgentOptions delete(config.OrgSettings, "secrets") // secrets are applied separately in Client.ApplyGroup - if _, ok := group.AppConfig.(map[string]interface{})["mdm"]; !ok { - group.AppConfig.(map[string]interface{})["mdm"] = map[string]interface{}{} + + // Integrations + var integrations interface{} + var ok bool + if integrations, ok = group.AppConfig.(map[string]interface{})["integrations"]; !ok || integrations == nil { + integrations = map[string]interface{}{} + group.AppConfig.(map[string]interface{})["integrations"] = integrations } + if jira, ok := integrations.(map[string]interface{})["jira"]; !ok || jira == nil { + integrations.(map[string]interface{})["jira"] = []interface{}{} + } + if zendesk, ok := integrations.(map[string]interface{})["zendesk"]; !ok || zendesk == nil { + integrations.(map[string]interface{})["zendesk"] = []interface{}{} + } + if googleCal, ok := integrations.(map[string]interface{})["google_calendar"]; !ok || googleCal == nil { + integrations.(map[string]interface{})["google_calendar"] = []interface{}{} + } + // Ensure mdm config exists mdmConfig, ok := group.AppConfig.(map[string]interface{})["mdm"] if !ok || mdmConfig == nil { @@ -941,6 +957,32 @@ func (c *Client) DoGitOps( // Clear out any existing host_status_webhook settings team["webhook_settings"].(map[string]interface{})["host_status_webhook"] = map[string]interface{}{} } + // Integrations + var integrations interface{} + var ok bool + if integrations, ok = config.TeamSettings["integrations"]; !ok || integrations == nil { + integrations = map[string]interface{}{} + } + team["integrations"] = integrations + _, ok = integrations.(map[string]interface{}) + if !ok { + return errors.New("team_settings.integrations config is not a map") + } + if calendar, ok := integrations.(map[string]interface{})["google_calendar"]; ok { + if calendar == nil { + calendar = map[string]interface{}{} + integrations.(map[string]interface{})["google_calendar"] = calendar + } + teamCalendarIntegration, ok = calendar.(map[string]interface{}) + if !ok { + return errors.New("team_settings.integrations.google_calendar config is not a map") + } + } + // We clear the calendar integration and re-apply it after updating policies. + // This is needed because the calendar integration may be referencing policies that need to be + // created/updated. + integrations.(map[string]interface{})["google_calendar"] = map[string]interface{}{} + team["mdm"] = map[string]interface{}{} mdmAppConfig = team["mdm"].(map[string]interface{}) } @@ -1044,6 +1086,24 @@ func (c *Client) DoGitOps( if err != nil { return err } + + // Apply calendar integration + if len(teamCalendarIntegration) > 0 { + group = spec.Group{} + team = make(map[string]interface{}) + team["name"] = *config.TeamName + team["integrations"] = map[string]interface{}{"google_calendar": teamCalendarIntegration} + rawTeam, err := json.Marshal(team) + if err != nil { + return fmt.Errorf("error marshalling team spec: %w", err) + } + group.Teams = []json.RawMessage{rawTeam} + _, err = c.ApplyGroup(ctx, &group, baseDir, logf, fleet.ApplySpecOptions{DryRun: dryRun}) + if err != nil { + return err + } + } + err = c.doGitOpsQueries(config, logFn, dryRun) if err != nil { return err diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 2367b7529..a02913a4e 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -5171,6 +5171,190 @@ func (s *integrationTestSuite) TestExternalIntegrationsConfig() { require.Len(t, config.Integrations.Zendesk, 0) } +func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { + t := s.T() + email := "service-account@example.com" + privateKey := "-----BEGIN PRIVATE KEY-----\nXXXXX\n-----END" + domain := "example.com" + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }] + } + }`, email, privateKey, domain, + )), http.StatusOK, + ) + + appConfig := s.getConfig() + require.Len(t, appConfig.Integrations.GoogleCalendar, 1) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) + assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) + + // Add 2nd config -- not allowed at this time + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }, + { + "email": "bozo@example.com"", + "private_key": "abc", + "domain": "example.com" + }] + } + }`, email, privateKey, domain, + )), http.StatusBadRequest, + ) + + // Make an unrelated config change, should not remove the integrations + var appCfgResp appConfigResponse + s.DoJSON( + "PATCH", "/api/v1/fleet/config", json.RawMessage( + `{ + "org_info": { + "org_name": "test-google-calendar-integrations" + } + }`, + ), http.StatusOK, &appCfgResp, + ) + require.Equal(t, "test-google-calendar-integrations", appCfgResp.OrgInfo.OrgName) + require.Len(t, appCfgResp.Integrations.GoogleCalendar, 1) + + // Update calendar config + domain = "new.com" + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }] + } + }`, email, privateKey, domain, + )), http.StatusOK, + ) + appConfig = s.getConfig() + require.Len(t, appConfig.Integrations.GoogleCalendar, 1) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) + assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) + + // Clearing other integrations does not clear Google Calendar integration + appCfgResp = appConfigResponse{} + s.DoJSON( + "PATCH", "/api/v1/fleet/config", json.RawMessage( + `{ + "integrations": { + "jira": [], + "zendesk": [] + } + }`, + ), http.StatusOK, &appCfgResp, + ) + require.Len(t, appCfgResp.Integrations.GoogleCalendar, 1) + + // Clearing Google Calendar integration + appCfgResp = appConfigResponse{} + s.DoJSON( + "PATCH", "/api/v1/fleet/config", json.RawMessage( + `{ + "integrations": { + "google_calendar": [] + } + }`, + ), http.StatusOK, &appCfgResp, + ) + assert.Empty(t, appCfgResp.Integrations.GoogleCalendar) + + // Try adding Google Calendar integration without sending private key -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "domain": %q + }] + } + }`, email, domain, + )), http.StatusUnprocessableEntity, + ) + + // Try adding Google Calendar integration with masked private key -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + }] + } + }`, email, fleet.MaskedPassword, domain, + )), http.StatusUnprocessableEntity, + ) + + // Empty email -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": " ", + "private_key": %q, + "domain": %q + }] + } + }`, privateKey, domain, + )), http.StatusUnprocessableEntity, + ) + + // Empty domain -- not allowed + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": "" + }] + } + }`, email, privateKey, + )), http.StatusUnprocessableEntity, + ) + + // Unknown fields fails as bad request + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": %q, + "domain": %q + "foo": "bar" + }] + } + }`, email, privateKey, domain, + )), http.StatusBadRequest, + ) + +} + func (s *integrationTestSuite) TestQueriesBadRequests() { t := s.T() diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 787207fb2..c9dfed256 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -97,6 +97,23 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { s.Do("POST", "/api/latest/fleet/teams", team, http.StatusOK) + // Create global calendar integration + calendarEmail := "service@example.com" + calendarWebhookUrl := "https://example.com/webhook" + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "email": %q, + "private_key": "testKey", + "domain": "example.com" + }] + } + }`, calendarEmail, + )), http.StatusOK, + ) + // updates a team, no secret is provided so it will keep the one generated // automatically when the team was created. agentOpts := json.RawMessage(`{"config": {"views": {"foo": "bar"}}, "overrides": {"platforms": {"darwin": {"views": {"bar": "qux"}}}}}`) @@ -163,6 +180,47 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { // an activity was created for team spec applied s.lastActivityMatches(fleet.ActivityTypeAppliedSpecTeam{}.ActivityName(), fmt.Sprintf(`{"teams": [{"id": %d, "name": %q}]}`, team.ID, team.Name), 0) + // Create team policy + teamPolicy, err := s.ds.NewTeamPolicy( + context.Background(), team.ID, nil, fleet.PolicyPayload{Name: "TestSpecTeamPolicy", Query: "SELECT 1"}, + ) + require.NoError(t, err) + defer func() { + _, err = s.ds.DeleteTeamPolicies(context.Background(), team.ID, []uint{teamPolicy.ID}) + require.NoError(t, err) + }() + + // Apply calendar integration + teamSpecs = map[string]any{ + "specs": []any{ + map[string]any{ + "name": teamName, + "integrations": map[string]any{ + "google_calendar": map[string]any{ + "email": calendarEmail, + "enable_calendar_events": true, + "policies": []any{ + map[string]any{ + "name": teamPolicy.Name, + }, + }, + "webhook_url": calendarWebhookUrl, + }, + }, + }, + }, + } + s.DoJSON("POST", "/api/latest/fleet/spec/teams", teamSpecs, http.StatusOK, &applyResp) + require.Len(t, applyResp.TeamIDsByName, 1) + + team, err = s.ds.TeamByName(context.Background(), teamName) + require.NotNil(t, team.Config.Integrations.GoogleCalendar) + assert.Equal(t, calendarEmail, team.Config.Integrations.GoogleCalendar.Email) + assert.Equal(t, calendarWebhookUrl, team.Config.Integrations.GoogleCalendar.WebhookURL) + assert.True(t, team.Config.Integrations.GoogleCalendar.Enable) + require.Len(t, team.Config.Integrations.GoogleCalendar.Policies, 1) + assert.Equal(t, teamPolicy.ID, team.Config.Integrations.GoogleCalendar.Policies[0].ID) + // dry-run with invalid windows updates teamSpecs = map[string]any{ "specs": []any{ diff --git a/tools/cloner-check/generated_files/appconfig.txt b/tools/cloner-check/generated_files/appconfig.txt index 5480e2432..7deb8d10f 100644 --- a/tools/cloner-check/generated_files/appconfig.txt +++ b/tools/cloner-check/generated_files/appconfig.txt @@ -84,6 +84,10 @@ github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration APIToken string github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration GroupID int64 github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableFailingPolicies bool github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableSoftwareVulnerabilities bool +github.com/fleetdm/fleet/v4/server/fleet/Integrations GoogleCalendar []*fleet.GoogleCalendarIntegration +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Email string +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration PrivateKey string +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Domain string github.com/fleetdm/fleet/v4/server/fleet/AppConfig MDM fleet.MDM github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMDefaultTeam string github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMEnabledAndConfigured bool From be0e89142f69a8b6ce636ec8b286be063667ecb9 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Wed, 13 Mar 2024 13:51:21 -0300 Subject: [PATCH 02/36] Add migrations for calendar events (#17585) #17230 --- server/datastore/mysql/hosts.go | 1 + server/datastore/mysql/hosts_test.go | 19 ++++++- .../20240313085226_AddCalendarEventTables.go | 52 ++++++++++++++++++ ...40313085226_AddCalendarEventTables_test.go | 53 +++++++++++++++++++ server/fleet/calendar_events.go | 29 ++++++++++ 5 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go create mode 100644 server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go create mode 100644 server/fleet/calendar_events.go diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index df2f4395b..ca8986e2e 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -502,6 +502,7 @@ var hostRefs = []string{ "query_results", "host_activities", "host_mdm_actions", + "host_calendar_events", } // NOTE: The following tables are explicity excluded from hostRefs list and accordingly are not diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index d1dedf061..b57784b1b 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -2554,7 +2554,6 @@ func testHostLiteByIdentifierAndID(t *testing.T, ds *Datastore) { h, err = ds.HostLiteByID(context.Background(), 0) assert.ErrorIs(t, err, sql.ErrNoRows) assert.Nil(t, h) - } func testHostsAddToTeam(t *testing.T, ds *Datastore) { @@ -2795,7 +2794,6 @@ func testHostsTotalAndUnseenSince(t *testing.T, ds *Datastore) { assert.Equal(t, 2, total) require.Len(t, unseen, 1) assert.Equal(t, host3.ID, unseen[0]) - } func testHostsListByPolicy(t *testing.T, ds *Datastore) { @@ -6577,6 +6575,23 @@ func testHostsDeleteHosts(t *testing.T, ds *Datastore) { `, host.ID) require.NoError(t, err) + // Add a calendar event for the host. + _, err = ds.writer(context.Background()).Exec(` + INSERT INTO calendar_events (email, start_time, end_time, event) + VALUES ('foobar@example.com', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, '{}'); + `) + require.NoError(t, err) + var calendarEventID int + err = ds.writer(context.Background()).Get(&calendarEventID, ` + SELECT id FROM calendar_events WHERE email = 'foobar@example.com'; + `) + require.NoError(t, err) + _, err = ds.writer(context.Background()).Exec(` + INSERT INTO host_calendar_events (host_id, calendar_event_id, webhook_status) + VALUES (?, ?, 1); + `, host.ID, calendarEventID) + require.NoError(t, err) + // Check there's an entry for the host in all the associated tables. for _, hostRef := range hostRefs { var ok bool diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go new file mode 100644 index 000000000..242a11ecb --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go @@ -0,0 +1,52 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20240313085226, Down_20240313085226) +} + +func Up_20240313085226(tx *sql.Tx) error { + // TODO(lucas): Check if we need more indexes. + + if _, err := tx.Exec(` + CREATE TABLE IF NOT EXISTS calendar_events ( + id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + email VARCHAR(255) NOT NULL, + start_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + end_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + event JSON NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ); +`); err != nil { + return fmt.Errorf("create calendar_events table: %w", err) + } + + if _, err := tx.Exec(` + CREATE TABLE IF NOT EXISTS host_calendar_events ( + id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + host_id INT(10) UNSIGNED NOT NULL, + calendar_event_id INT(10) UNSIGNED NOT NULL, + webhook_status TINYINT NOT NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + UNIQUE KEY idx_one_calendar_event_per_host (host_id), + FOREIGN KEY (calendar_event_id) REFERENCES calendar_events(id) ON DELETE CASCADE + ); +`); err != nil { + return fmt.Errorf("create host_calendar_events table: %w", err) + } + + return nil +} + +func Down_20240313085226(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go new file mode 100644 index 000000000..216a3e468 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go @@ -0,0 +1,53 @@ +package tables + +import ( + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestUp_20240313085226(t *testing.T) { + db := applyUpToPrev(t) + applyNext(t, db) + + sampleEvent := fleet.CalendarEvent{ + Email: "foo@example.com", + StartTime: time.Now().UTC(), + EndTime: time.Now().UTC().Add(30 * time.Minute), + Data: []byte("{\"foo\": \"bar\"}"), + } + sampleEvent.ID = uint(execNoErrLastID(t, db, + `INSERT INTO calendar_events (email, start_time, end_time, event) VALUES (?, ?, ?, ?);`, + sampleEvent.Email, sampleEvent.StartTime, sampleEvent.EndTime, sampleEvent.Data, + )) + + sampleHostEvent := fleet.HostCalendarEvent{ + HostID: 1, + CalendarEventID: sampleEvent.ID, + WebhookStatus: fleet.CalendarWebhookStatusPending, + } + sampleHostEvent.ID = uint(execNoErrLastID(t, db, + `INSERT INTO host_calendar_events (host_id, calendar_event_id, webhook_status) VALUES (?, ?, ?);`, + sampleHostEvent.HostID, sampleHostEvent.CalendarEventID, sampleHostEvent.WebhookStatus, + )) + + var event fleet.CalendarEvent + err := db.Get(&event, `SELECT * FROM calendar_events WHERE id = ?;`, sampleEvent.ID) + require.NoError(t, err) + sampleEvent.CreatedAt = event.CreatedAt // sampleEvent doesn't have this set. + sampleEvent.UpdatedAt = event.UpdatedAt // sampleEvent doesn't have this set. + sampleEvent.StartTime = sampleEvent.StartTime.Round(time.Second) + sampleEvent.EndTime = sampleEvent.EndTime.Round(time.Second) + event.StartTime = event.StartTime.Round(time.Second) + event.EndTime = event.EndTime.Round(time.Second) + require.Equal(t, sampleEvent, event) + + var hostEvent fleet.HostCalendarEvent + err = db.Get(&hostEvent, `SELECT * FROM host_calendar_events WHERE id = ?;`, sampleHostEvent.ID) + require.NoError(t, err) + sampleHostEvent.CreatedAt = hostEvent.CreatedAt // sampleHostEvent doesn't have this set. + sampleHostEvent.UpdatedAt = hostEvent.UpdatedAt // sampleHostEvent doesn't have this set. + require.Equal(t, sampleHostEvent, hostEvent) +} diff --git a/server/fleet/calendar_events.go b/server/fleet/calendar_events.go new file mode 100644 index 000000000..7671b4aba --- /dev/null +++ b/server/fleet/calendar_events.go @@ -0,0 +1,29 @@ +package fleet + +import "time" + +type CalendarEvent struct { + ID uint `db:"id"` + Email string `db:"email"` + StartTime time.Time `db:"start_time"` + EndTime time.Time `db:"end_time"` + Data []byte `db:"event"` + + UpdateCreateTimestamps +} + +type CalendarWebhookStatus int + +const ( + CalendarWebhookStatusPending CalendarWebhookStatus = iota + CalendarWebhookStatusSent +) + +type HostCalendarEvent struct { + ID uint `db:"id"` + HostID uint `db:"host_id"` + CalendarEventID uint `db:"calendar_event_id"` + WebhookStatus CalendarWebhookStatus `db:"webhook_status"` + + UpdateCreateTimestamps +} From c9b917a49105110964ef35bc7777f9e6d908aed5 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 14:07:13 -0500 Subject: [PATCH 03/36] Calendar interface (#17633) # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features. - [ ] Added/updated tests - [x] Manual QA for all new/changed functionality --- ee/server/calendar/calendar.go | 16 ++ ee/server/calendar/google_calendar.go | 393 ++++++++++++++++++++++++++ server/fleet/calendar.go | 24 ++ 3 files changed, 433 insertions(+) create mode 100644 ee/server/calendar/calendar.go create mode 100644 ee/server/calendar/google_calendar.go create mode 100644 server/fleet/calendar.go diff --git a/ee/server/calendar/calendar.go b/ee/server/calendar/calendar.go new file mode 100644 index 000000000..469437f3a --- /dev/null +++ b/ee/server/calendar/calendar.go @@ -0,0 +1,16 @@ +package calendar + +import ( + "context" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/kit/log" +) + +type GoogleCalendarConfig struct { + Context context.Context + IntegrationConfig *fleet.GoogleCalendarIntegration + UserEmail string + Logger log.Logger + // Should be nil for production + API GoogleCalendarAPI +} diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go new file mode 100644 index 000000000..ff3d2dfb8 --- /dev/null +++ b/ee/server/calendar/google_calendar.go @@ -0,0 +1,393 @@ +package calendar + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/log/level" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" + "google.golang.org/api/option" + "net/http" + "time" +) + +const ( + eventTitle = "💻🚫Downtime" + startHour = 9 + endHour = 17 + eventLength = 30 * time.Minute + calendarID = "primary" +) + +var calendarScopes = []string{ + "https://www.googleapis.com/auth/calendar.events", + "https://www.googleapis.com/auth/calendar.settings.readonly", +} + +// GoogleCalendar is an implementation of the Calendar interface that uses the +// Google Calendar API to manage events. +type GoogleCalendar struct { + config *GoogleCalendarConfig + timezoneOffset *int +} + +type GoogleCalendarAPI interface { + Connect(ctx context.Context, email, privateKey, subject string) error + GetSetting(name string) (*calendar.Setting, error) + ListEvents(timeMin, timeMax string) (*calendar.Events, error) + CreateEvent(event *calendar.Event) (*calendar.Event, error) + GetEvent(id, eTag string) (*calendar.Event, error) + DeleteEvent(id string) error +} + +type eventDetails struct { + ID string `json:"id"` + ETag string `json:"etag"` +} + +type GoogleCalendarLowLevelAPI struct { + service *calendar.Service +} + +// Connect creates a new Google Calendar service using the provided credentials. +func (lowLevelAPI *GoogleCalendarLowLevelAPI) Connect(ctx context.Context, email, privateKey, subject string) error { + // Create a new calendar service + conf := &jwt.Config{ + Email: email, + Scopes: calendarScopes, + PrivateKey: []byte(privateKey), + TokenURL: google.JWTTokenURL, + Subject: subject, + } + client := conf.Client(ctx) + service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + return err + } + lowLevelAPI.service = service + return nil +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) { + return lowLevelAPI.service.Settings.Get(name).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + return lowLevelAPI.service.Events.Insert(calendarID, event).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calendar.Event, error) { + return lowLevelAPI.service.Events.Get(calendarID, id).IfNoneMatch(eTag).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { + // Default maximum number of events returned is 250, which should be sufficient for most calendars. + return lowLevelAPI.service.Events.List(calendarID).EventTypes("default").OrderBy("startTime").SingleEvents(true).TimeMin(timeMin).TimeMax(timeMax).Do() +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { + return lowLevelAPI.service.Events.Delete(calendarID, id).Do() +} + +func (c *GoogleCalendar) Connect(config any) (fleet.Calendar, error) { + gConfig, ok := config.(*GoogleCalendarConfig) + if !ok { + return nil, errors.New("invalid Google calendar config") + } + if gConfig.API == nil { + var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} + gConfig.API = lowLevelAPI + } + err := gConfig.API.Connect( + gConfig.Context, gConfig.IntegrationConfig.Email, gConfig.IntegrationConfig.PrivateKey, gConfig.UserEmail, + ) + if err != nil { + return nil, ctxerr.Wrap(gConfig.Context, err, "creating Google calendar service") + } + + gCal := &GoogleCalendar{ + config: gConfig, + } + + return gCal, nil +} + +func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { + if c.config == nil { + return nil, false, errors.New("the Google calendar is not connected. Please call Connect first") + } + if event.EndTime.Before(time.Now()) { + return nil, false, ctxerr.Errorf(c.config.Context, "cannot get and update an event that has already ended: %s", event.EndTime) + } + details, err := c.unmarshalDetails(event) + if err != nil { + return nil, false, err + } + gEvent, err := c.config.API.GetEvent(details.ID, details.ETag) + var deleted bool + switch { + // http.StatusNotModified is returned sometimes, but not always, so we need to check ETag explicitly later + case googleapi.IsNotModified(err): + return event, false, nil + case isNotFound(err): + deleted = true + case err != nil: + return nil, false, ctxerr.Wrap(c.config.Context, err, "retrieving Google calendar event") + } + if !deleted && gEvent.Status != "cancelled" { + if details.ETag == gEvent.Etag { + // Event was not modified + return event, false, nil + } + endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) + if err != nil { + return nil, false, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), + ) + } + // If event already ended, it is effectively deleted + if endTime.After(time.Now()) { + startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + if err != nil { + return nil, false, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), + ) + } + fleetEvent, err := c.googleEventToFleetEvent(startTime, endTime, gEvent) + if err != nil { + return nil, false, err + } + return fleetEvent, true, nil + } + } + + newStartDate := event.StartTime.Add(24 * time.Hour) + if newStartDate.Weekday() == time.Saturday { + newStartDate = newStartDate.Add(48 * time.Hour) + } else if newStartDate.Weekday() == time.Sunday { + newStartDate = newStartDate.Add(24 * time.Hour) + } + + fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn()) + if err != nil { + return nil, false, err + } + return fleetEvent, true, nil +} + +func isNotFound(err error) bool { + if err == nil { + return false + } + var ae *googleapi.Error + ok := errors.As(err, &ae) + return ok && ae.Code == http.StatusNotFound +} + +func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) { + var details eventDetails + err := json.Unmarshal(event.Data, &details) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "unmarshaling Google calendar event details") + } + if details.ID == "" { + return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ID") + } + if details.ETag == "" { + return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ETag") + } + return &details, nil +} + +func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { + if c.config == nil { + return nil, errors.New("the Google calendar is not connected. Please call Connect first") + } + if c.timezoneOffset == nil { + err := getTimezone(c) + if err != nil { + return nil, err + } + } + + location := time.FixedZone("", *c.timezoneOffset) + dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, location) + dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, location) + + now := time.Now().In(location) + if dayEnd.Before(now) { + // The workday has already ended. + return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "cannot schedule an event for a day that has already ended"}) + } + + // Adjust day start if workday already started + if dayStart.Before(now) { + dayStart = now.Truncate(eventLength) + if dayStart.Before(now) { + dayStart = dayStart.Add(eventLength) + } + if dayStart.Equal(dayEnd) { + return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "no time available for event"}) + } + } + eventStart := dayStart + eventEnd := dayStart.Add(eventLength) + + searchStart := dayStart.Add(-24 * time.Hour) + events, err := c.config.API.ListEvents(searchStart.Format(time.RFC3339), dayEnd.Format(time.RFC3339)) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "listing Google calendar events") + } + for _, gEvent := range events.Items { + // Ignore cancelled events + if gEvent.Status == "cancelled" { + continue + } + + // Ignore events that the user has declined + var attending bool + if len(gEvent.Attendees) == 0 { + // No attendees, so we assume the user is attending + attending = true + } else { + for _, attendee := range gEvent.Attendees { + if attendee.Email == c.config.UserEmail { + if attendee.ResponseStatus != "declined" { + attending = true + } + break + } + } + } + if !attending { + continue + } + + // Ignore events that will end before our event + endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) + if err != nil { + return nil, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), + ) + } + if endTime.Before(eventStart) || endTime.Equal(eventStart) { + continue + } + + startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + if err != nil { + return nil, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), + ) + } + + if startTime.Before(eventEnd) { + // Event occurs during our event, so we need to adjust. + fmt.Printf("VICTOR Adjusting event times due to %s: %s - %s\n", gEvent.Summary, eventStart, eventEnd) + var isLastSlot bool + eventStart, eventEnd, isLastSlot = adjustEventTimes(endTime, dayEnd) + if isLastSlot { + break + } + continue + } + // Since events are sorted by startTime, all subsequent events are after our event, so we can stop processing + break + } + + event := &calendar.Event{} + event.Start = &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)} + event.End = &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)} + event.Summary = eventTitle + event.Description = body + event, err = c.config.API.CreateEvent(event) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "creating Google calendar event") + } + + // Convert Google event to Fleet event + fleetEvent, err := c.googleEventToFleetEvent(eventStart, eventEnd, event) + if err != nil { + return nil, err + } + level.Debug(c.config.Logger).Log("msg", "created Google calendar events", "user", c.config.UserEmail, "startTime", eventStart) + fmt.Printf("VICTOR Created event with id:%s and ETag:%s\n", event.Id, event.Etag) + + return fleetEvent, nil +} + +func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time, eventEnd time.Time, isLastSlot bool) { + eventStart = endTime.Truncate(eventLength) + if eventStart.Before(endTime) { + eventStart = eventStart.Add(eventLength) + } + eventEnd = eventStart.Add(eventLength) + // If we are at the end of the day, pick the last slot + if eventEnd.After(dayEnd) { + eventEnd = dayEnd + eventStart = eventEnd.Add(-eventLength) + isLastSlot = true + } + if eventEnd.Equal(dayEnd) { + isLastSlot = true + } + return eventStart, eventEnd, isLastSlot +} + +func getTimezone(gCal *GoogleCalendar) error { + config := gCal.config + setting, err := config.API.GetSetting("timezone") + if err != nil { + return ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone") + } + + loc, err := time.LoadLocation(setting.Value) + if err != nil { + // Could not load location, use EST + level.Warn(config.Logger).Log("msg", "parsing Google calendar timezone", "timezone", setting.Value, "err", err) + loc, _ = time.LoadLocation("America/New_York") + } + _, timezoneOffset := time.Now().In(loc).Zone() + gCal.timezoneOffset = &timezoneOffset + return nil +} + +func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime time.Time, event *calendar.Event) ( + *fleet.CalendarEvent, error, +) { + fleetEvent := &fleet.CalendarEvent{} + fleetEvent.StartTime = startTime + fleetEvent.EndTime = endTime + fleetEvent.Email = c.config.UserEmail + details := &eventDetails{ + ID: event.Id, + ETag: event.Etag, + } + detailsJson, err := json.Marshal(details) + if err != nil { + return nil, ctxerr.Wrap(c.config.Context, err, "marshaling Google calendar event details") + } + fleetEvent.Data = detailsJson + return fleetEvent, nil +} + +func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error { + if c.config == nil { + return errors.New("the Google calendar is not connected. Please call Connect first") + } + details, err := c.unmarshalDetails(event) + if err != nil { + return err + } + err = c.config.API.DeleteEvent(details.ID) + if err != nil { + return ctxerr.Wrap(c.config.Context, err, "deleting Google calendar event") + } + return nil +} diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go new file mode 100644 index 000000000..dc682e24e --- /dev/null +++ b/server/fleet/calendar.go @@ -0,0 +1,24 @@ +package fleet + +import "time" + +type DayEndedError struct { + Msg string +} + +func (e DayEndedError) Error() string { + return e.Msg +} + +type Calendar interface { + // Connect to calendar. This method must be called first. Currently, config must be a *GoogleCalendarConfig + Connect(config any) (Calendar, error) + // GetAndUpdateEvent retrieves the event from the calendar. + // If the event has been modified, it returns the updated event. + // If the event has been deleted, it schedules a new event with given body callback and returns the new event. + GetAndUpdateEvent(event *CalendarEvent, genBodyFn func() string) (updatedEvent *CalendarEvent, updated bool, err error) + // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. + CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) + // DeleteEvent deletes the event with the given ID. + DeleteEvent(event *CalendarEvent) error +} From d3e1716572f8432627649d3735b80246b1e869b5 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 15:01:52 -0500 Subject: [PATCH 04/36] Calendar config API endpoints bug fixes. (#17640) Bug fixes for frontend - google_calendar can be nil for global config to indicate that it should not change - `fleet/teams/:id` endpoint now working --- ee/server/service/teams.go | 9 +++++++++ server/service/appconfig.go | 4 ++++ server/service/integration_enterprise_test.go | 15 +++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index f0ad454df..f573c89fa 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -209,6 +209,15 @@ func (svc *Service) ModifyTeam(ctx context.Context, teamID uint, payload fleet.T team.Config.Integrations.Jira = payload.Integrations.Jira team.Config.Integrations.Zendesk = payload.Integrations.Zendesk + // Only update the google calendar integration if it's not nil + if payload.Integrations.GoogleCalendar != nil { + invalid := &fleet.InvalidArgumentError{} + _ = svc.validateTeamCalendarIntegrations(ctx, team, payload.Integrations.GoogleCalendar, appCfg, invalid) + if invalid.HasErrors() { + return nil, ctxerr.Wrap(ctx, invalid) + } + team.Config.Integrations.GoogleCalendar = payload.Integrations.GoogleCalendar + } } if payload.WebhookSettings != nil || payload.Integrations != nil { diff --git a/server/service/appconfig.go b/server/service/appconfig.go index 88ee842e3..8a346183d 100644 --- a/server/service/appconfig.go +++ b/server/service/appconfig.go @@ -478,6 +478,10 @@ func (svc *Service) ModifyAppConfig(ctx context.Context, p []byte, applyOpts fle } } } + // If google_calendar is null, we keep the existing setting. If it's not null, we update. + if newAppConfig.Integrations.GoogleCalendar == nil { + appConfig.Integrations.GoogleCalendar = oldAppConfig.Integrations.GoogleCalendar + } if !license.IsPremium() { // reset transparency url to empty for downgraded licenses diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index c9dfed256..2484ce0d1 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -1029,6 +1029,21 @@ func (s *integrationEnterpriseTestSuite) TestTeamEndpoints() { modifyExpiry.HostExpirySettings.HostExpiryWindow = 0 s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/teams/%d", tm1ID), modifyExpiry, http.StatusUnprocessableEntity, &tmResp) + // Modify team's calendar config + modifyCalendar := fleet.TeamPayload{ + Integrations: &fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Email: "calendar@example.com", + }, + }, + } + s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/teams/%d", tm1ID), modifyCalendar, http.StatusOK, &tmResp) + assert.Equal(t, modifyCalendar.Integrations.GoogleCalendar, tmResp.Team.Config.Integrations.GoogleCalendar) + + // Illegal team calendar config + modifyCalendar.Integrations.GoogleCalendar.Enable = true + s.DoJSON("PATCH", fmt.Sprintf("/api/latest/fleet/teams/%d", tm1ID), modifyCalendar, http.StatusUnprocessableEntity, &tmResp) + // list team users var usersResp listUsersResponse s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d/users", tm1ID), nil, http.StatusOK, &usersResp) From 63e9d49dfc687741376ec9a4c1e3e4a1db60146c Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 19:00:51 -0500 Subject: [PATCH 05/36] Calendar config updates -- policy table now has calendar_events_enabled (#17645) # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [x] Added/updated tests - [x] If database migrations are included, checked table schema to confirm autoupdate - For database migrations: - [x] Checked schema for all modified table for columns that will auto-update timestamps during migration. - [x] Confirmed that updating the timestamps is acceptable, and will not cause unwanted side effects. - [x] Manual QA for all new/changed functionality --- cmd/fleetctl/apply_test.go | 41 ------- cmd/fleetctl/get_test.go | 38 ++++--- cmd/fleetctl/gitops_test.go | 7 -- .../expectedHostDetailResponseJson.json | 6 +- .../expectedHostDetailResponseYaml.yml | 2 + .../testdata/gitops/team_config_no_paths.yml | 4 +- ee/server/service/teams.go | 53 +++------ ...40314151747_AddCalendarEventsToPolicies.go | 22 ++++ ...151747_AddCalendarEventsToPolicies_test.go | 42 +++++++ server/datastore/mysql/policies.go | 55 ++------- server/datastore/mysql/policies_test.go | 107 +++++++----------- server/fleet/datastore.go | 1 - server/fleet/integrations.go | 13 +-- server/fleet/policies.go | 8 ++ server/fleet/teams.go | 6 + server/mock/datastore_mock.go | 12 -- server/service/client.go | 32 +----- server/service/integration_enterprise_test.go | 63 +++++------ server/service/team_policies.go | 35 +++--- server/webhooks/failing_policies_test.go | 27 +++-- 20 files changed, 241 insertions(+), 333 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go create mode 100644 server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index f53a78313..655e2e57b 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -452,20 +452,6 @@ spec: ) // Apply calendar integration - validPolicyID := uint(10) - validPolicyName := "validPolicy" - ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - var policies = make(map[string]*fleet.Policy) - for _, name := range names { - if name != validPolicyName { - return nil, ¬FoundError{} - } - policies[name] = &fleet.Policy{ - PolicyData: fleet.PolicyData{ID: validPolicyID, TeamID: &teamsByName["team1"].ID, Name: validPolicyName}, - } - } - return policies, nil - } filename = writeTmpYml( t, ` apiVersion: v1 @@ -477,8 +463,6 @@ spec: google_calendar: email: `+googleCalEmail+` enable_calendar_events: true - policies: - - name: `+validPolicyName+` webhook_url: https://example.com/webhook `, ) @@ -488,7 +472,6 @@ spec: t, fleet.TeamGoogleCalendarIntegration{ Email: googleCalEmail, Enable: true, - Policies: []*fleet.PolicyRef{{Name: validPolicyName, ID: validPolicyID}}, WebhookURL: "https://example.com/webhook", }, *teamsByName["team1"].Config.Integrations.GoogleCalendar, ) @@ -505,8 +488,6 @@ spec: google_calendar: email: not_present_globally@example.com enable_calendar_events: true - policies: - - name: `+validPolicyName+` webhook_url: https://example.com/webhook `, ) @@ -514,26 +495,6 @@ spec: _, err = runAppNoChecks([]string{"apply", "-f", filename}) assert.ErrorContains(t, err, "email must match a global Google Calendar integration email") - // Apply calendar integration -- invalid policy name - filename = writeTmpYml( - t, ` -apiVersion: v1 -kind: team -spec: - team: - name: team1 - integrations: - google_calendar: - email: `+googleCalEmail+` - enable_calendar_events: true - policies: - - name: invalidPolicy - webhook_url: https://example.com/webhook -`, - ) - _, err = runAppNoChecks([]string{"apply", "-f", filename}) - assert.ErrorContains(t, err, "name is invalid") - // Apply calendar integration -- invalid webhook destination filename = writeTmpYml( t, ` @@ -546,8 +507,6 @@ spec: google_calendar: email: `+googleCalEmail+` enable_calendar_events: true - policies: - - name: `+validPolicyName+` webhook_url: bozo `, ) diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 8bf99d774..07bb5b3e1 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -331,29 +331,31 @@ func TestGetHosts(t *testing.T) { return []*fleet.HostPolicy{ { PolicyData: fleet.PolicyData{ - ID: 1, - Name: "query1", - Query: defaultPolicyQuery, - Description: "Some description", - AuthorID: ptr.Uint(1), - AuthorName: "Alice", - AuthorEmail: "alice@example.com", - Resolution: ptr.String("Some resolution"), - TeamID: ptr.Uint(1), + ID: 1, + Name: "query1", + Query: defaultPolicyQuery, + Description: "Some description", + AuthorID: ptr.Uint(1), + AuthorName: "Alice", + AuthorEmail: "alice@example.com", + Resolution: ptr.String("Some resolution"), + TeamID: ptr.Uint(1), + CalendarEventsEnabled: true, }, Response: "passes", }, { PolicyData: fleet.PolicyData{ - ID: 2, - Name: "query2", - Query: defaultPolicyQuery, - Description: "", - AuthorID: ptr.Uint(1), - AuthorName: "Alice", - AuthorEmail: "alice@example.com", - Resolution: nil, - TeamID: nil, + ID: 2, + Name: "query2", + Query: defaultPolicyQuery, + Description: "", + AuthorID: ptr.Uint(1), + AuthorName: "Alice", + AuthorEmail: "alice@example.com", + Resolution: nil, + TeamID: nil, + CalendarEventsEnabled: false, }, Response: "fails", }, diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index 048c1750a..c6013aa95 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -466,12 +466,6 @@ func TestFullTeamGitOps(t *testing.T) { } return nil, nil } - ds.PoliciesByNameFunc = func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - if slices.Contains(names, "policy1") && slices.Contains(names, "policy2") { - return map[string]*fleet.Policy{"policy1": &policy, "policy2": &policy}, nil - } - return nil, nil - } ds.DeleteTeamPoliciesFunc = func(ctx context.Context, teamID uint, IDs []uint) ([]uint, error) { policyDeleted = true assert.Equal(t, []uint{policy.ID}, IDs) @@ -554,7 +548,6 @@ func TestFullTeamGitOps(t *testing.T) { require.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) assert.Equal(t, "service@example.com", savedTeam.Config.Integrations.GoogleCalendar.Email) assert.True(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) - assert.Len(t, savedTeam.Config.Integrations.GoogleCalendar.Policies, 2) // Now clear the settings tmpFile, err := os.CreateTemp(t.TempDir(), "*.yml") diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json index 8c6e8dc3a..0a05fb8b0 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseJson.json @@ -76,7 +76,8 @@ "team_id": 1, "updated_at": "0001-01-01T00:00:00Z", "created_at": "0001-01-01T00:00:00Z", - "critical": false + "critical": false, + "calendar_events_enabled": true }, { "id": 2, @@ -91,7 +92,8 @@ "team_id": null, "updated_at": "0001-01-01T00:00:00Z", "created_at": "0001-01-01T00:00:00Z", - "critical": false + "critical": false, + "calendar_events_enabled": false } ], "status": "offline", diff --git a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml index fc4431c9e..5b1f81d4a 100644 --- a/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml +++ b/cmd/fleetctl/testdata/expectedHostDetailResponseYaml.yml @@ -62,6 +62,7 @@ spec: created_at: "0001-01-01T00:00:00Z" updated_at: "0001-01-01T00:00:00Z" critical: false + calendar_events_enabled: true - author_email: "alice@example.com" author_id: 1 author_name: Alice @@ -75,6 +76,7 @@ spec: created_at: "0001-01-01T00:00:00Z" updated_at: "0001-01-01T00:00:00Z" critical: false + calendar_events_enabled: false policy_updated_at: "0001-01-01T00:00:00Z" public_ip: "" primary_ip: "" diff --git a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml index 608bafcf7..58564a7bc 100644 --- a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml @@ -19,9 +19,6 @@ team_settings: google_calendar: email: service@example.com enable_calendar_events: true - policies: - - name: policy1 - - name: policy2 webhook_url: https://example.com/google_calendar_webhook agent_options: command_line_flags: @@ -97,6 +94,7 @@ policies: description: This policy should always fail. resolution: There is no resolution for this policy. query: SELECT 1 FROM osquery_info WHERE start_time < 0; + calendar_events_enabled: true - name: Passing policy platform: linux,windows,darwin,chrome description: This policy should always pass. diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index f573c89fa..e0f503b38 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -197,19 +197,21 @@ func (svc *Service) ModifyTeam(ctx context.Context, teamID uint, payload fleet.T } if payload.Integrations != nil { - // the team integrations must reference an existing global config integration. - if _, err := payload.Integrations.MatchWithIntegrations(appCfg.Integrations); err != nil { - return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) - } + if payload.Integrations.Jira != nil || payload.Integrations.Zendesk != nil { + // the team integrations must reference an existing global config integration. + if _, err := payload.Integrations.MatchWithIntegrations(appCfg.Integrations); err != nil { + return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) + } - // integrations must be unique - if err := payload.Integrations.Validate(); err != nil { - return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) - } + // integrations must be unique + if err := payload.Integrations.Validate(); err != nil { + return nil, fleet.NewInvalidArgumentError("integrations", err.Error()) + } - team.Config.Integrations.Jira = payload.Integrations.Jira - team.Config.Integrations.Zendesk = payload.Integrations.Zendesk - // Only update the google calendar integration if it's not nil + team.Config.Integrations.Jira = payload.Integrations.Jira + team.Config.Integrations.Zendesk = payload.Integrations.Zendesk + } + // Only update the calendar integration if it's not nil if payload.Integrations.GoogleCalendar != nil { invalid := &fleet.InvalidArgumentError{} _ = svc.validateTeamCalendarIntegrations(ctx, team, payload.Integrations.GoogleCalendar, appCfg, invalid) @@ -1179,35 +1181,6 @@ func (svc *Service) validateTeamCalendarIntegrations( } else if u.Scheme != "https" && u.Scheme != "http" { invalid.Append("integrations.google_calendar.webhook_url", "webhook_url must be https or http") } - // Validate policy ids - if len(calendarIntegration.Policies) == 0 { - invalid.Append("integrations.google_calendar.policies", "policies are required") - } - if len(calendarIntegration.Policies) > 0 { - for _, policy := range calendarIntegration.Policies { - policy.Name = strings.TrimSpace(policy.Name) - } - calendarIntegration.Policies = server.RemoveDuplicatesFromSlice(calendarIntegration.Policies) - policyNames := make([]string, 0, len(calendarIntegration.Policies)) - for _, policy := range calendarIntegration.Policies { - policyNames = append(policyNames, policy.Name) - } - // Policies must be team policies. Global policies are not allowed. - policyMap, err := svc.ds.PoliciesByName(ctx, policyNames, team.ID) - if err != nil { - level.Error(svc.logger).Log("msg", "error getting policies by name", "names", policyNames, "err", err) - if fleet.IsNotFound(err) { - invalid.Append("integrations.google_calendar.policies[].name", "name is invalid") - } else { - return err - } - } else { - // PoliciesByName guarantees that all policies are present - for _, policy := range calendarIntegration.Policies { - policy.ID = policyMap[policy.Name].ID - } - } - } return nil } diff --git a/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go new file mode 100644 index 000000000..485993bf7 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies.go @@ -0,0 +1,22 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20240314151747, Down_20240314151747) +} + +func Up_20240314151747(tx *sql.Tx) error { + _, err := tx.Exec(`ALTER TABLE policies ADD COLUMN calendar_events_enabled TINYINT(1) UNSIGNED NOT NULL DEFAULT '0'`) + if err != nil { + return fmt.Errorf("failed to add calendar_events_enabled to policies: %w", err) + } + return nil +} + +func Down_20240314151747(_ *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go new file mode 100644 index 000000000..2deb81a9f --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240314151747_AddCalendarEventsToPolicies_test.go @@ -0,0 +1,42 @@ +package tables + +import ( + "context" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestUp_20240314151747(t *testing.T) { + db := applyUpToPrev(t) + + policy1 := execNoErrLastID( + t, db, "INSERT INTO policies (name, query, description, checksum) VALUES (?,?,?,?)", "policy", "", "", "checksum", + ) + + // Apply current migration. + applyNext(t, db) + + var policyCheck []struct { + ID int64 `db:"id"` + CalEnabled bool `db:"calendar_events_enabled"` + } + err := db.SelectContext(context.Background(), &policyCheck, `SELECT id, calendar_events_enabled FROM policies ORDER BY id`) + require.NoError(t, err) + require.Len(t, policyCheck, 1) + assert.Equal(t, policy1, policyCheck[0].ID) + assert.Equal(t, false, policyCheck[0].CalEnabled) + + policy2 := execNoErrLastID( + t, db, "INSERT INTO policies (name, query, description, checksum, calendar_events_enabled) VALUES (?,?,?,?,?)", "policy2", "", "", + "checksum2", 1, + ) + + policyCheck = nil + err = db.SelectContext(context.Background(), &policyCheck, `SELECT id, calendar_events_enabled FROM policies WHERE id = ?`, policy2) + require.NoError(t, err) + require.Len(t, policyCheck, 1) + assert.Equal(t, policy2, policyCheck[0].ID) + assert.Equal(t, true, policyCheck[0].CalEnabled) + +} diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 39c7d3e05..d2f242407 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "encoding/json" - "errors" "fmt" "golang.org/x/text/unicode/norm" "sort" @@ -20,7 +19,7 @@ import ( const policyCols = ` p.id, p.team_id, p.resolution, p.name, p.query, p.description, - p.author_id, p.platforms, p.created_at, p.updated_at, p.critical + p.author_id, p.platforms, p.created_at, p.updated_at, p.critical, p.calendar_events_enabled ` var policySearchColumns = []string{"p.name"} @@ -116,10 +115,12 @@ func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemo p.Name = norm.NFC.String(p.Name) sql := ` UPDATE policies - SET name = ?, query = ?, description = ?, resolution = ?, platforms = ?, critical = ?, checksum = ` + policiesChecksumComputedColumn() + ` + SET name = ?, query = ?, description = ?, resolution = ?, platforms = ?, critical = ?, calendar_events_enabled = ?, checksum = ` + policiesChecksumComputedColumn() + ` WHERE id = ? ` - result, err := ds.writer(ctx).ExecContext(ctx, sql, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.ID) + result, err := ds.writer(ctx).ExecContext( + ctx, sql, p.Name, p.Query, p.Description, p.Resolution, p.Platform, p.Critical, p.CalendarEventsEnabled, p.ID, + ) if err != nil { return ctxerr.Wrap(ctx, err, "updating policy") } @@ -445,42 +446,6 @@ func (ds *Datastore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fl return policiesByID, nil } -func (ds *Datastore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - sqlQuery := `SELECT ` + policyCols + ` - FROM policies p - WHERE p.team_id = ? AND p.name IN (?)` - query, args, err := sqlx.In(sqlQuery, teamID, names) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building query to get policies by name") - } - - var policies []*fleet.Policy - err = sqlx.SelectContext( - ctx, - ds.reader(ctx), - &policies, - query, args..., - ) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(fmt.Sprintf("%v", names))) - } - return nil, ctxerr.Wrap(ctx, err, "getting policies by name") - } - - policiesByName := make(map[string]*fleet.Policy, len(names)) - for _, p := range policies { - policiesByName[p.Name] = p - } - for _, name := range names { - if policiesByName[name] == nil { - return nil, ctxerr.Wrap(ctx, notFound("Policy").WithName(name)) - } - } - - return policiesByName, nil -} - func (ds *Datastore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { return deletePolicyDB(ctx, ds.writer(ctx), ids, nil) } @@ -562,10 +527,11 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u nameUnicode := norm.NFC.String(args.Name) res, err := ds.writer(ctx).ExecContext(ctx, fmt.Sprintf( - `INSERT INTO policies (name, query, description, team_id, resolution, author_id, platforms, critical, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, %s)`, + `INSERT INTO policies (name, query, description, team_id, resolution, author_id, platforms, critical, calendar_events_enabled, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, %s)`, policiesChecksumComputedColumn(), ), nameUnicode, args.Query, args.Description, teamID, args.Resolution, authorID, args.Platform, args.Critical, + args.CalendarEventsEnabled, ) switch { case err == nil: @@ -623,15 +589,17 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs team_id, platforms, critical, + calendar_events_enabled, checksum - ) VALUES ( ?, ?, ?, ?, ?, (SELECT IFNULL(MIN(id), NULL) FROM teams WHERE name = ?), ?, ?, %s) + ) VALUES ( ?, ?, ?, ?, ?, (SELECT IFNULL(MIN(id), NULL) FROM teams WHERE name = ?), ?, ?, ?, %s) ON DUPLICATE KEY UPDATE query = VALUES(query), description = VALUES(description), author_id = VALUES(author_id), resolution = VALUES(resolution), platforms = VALUES(platforms), - critical = VALUES(critical) + critical = VALUES(critical), + calendar_events_enabled = VALUES(calendar_events_enabled) `, policiesChecksumComputedColumn(), ) for _, spec := range specs { @@ -640,6 +608,7 @@ func (ds *Datastore) ApplyPolicySpecs(ctx context.Context, authorID uint, specs spec.Name = norm.NFC.String(spec.Name) res, err := tx.ExecContext(ctx, query, spec.Name, spec.Query, spec.Description, authorID, spec.Resolution, spec.Team, spec.Platform, spec.Critical, + spec.CalendarEventsEnabled, ) if err != nil { return ctxerr.Wrap(ctx, err, "exec ApplyPolicySpecs insert") diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index 52da13348..b0ef3b1bc 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -38,7 +38,6 @@ func TestPolicies(t *testing.T) { {"PolicyQueriesForHost", testPolicyQueriesForHost}, {"PolicyQueriesForHostPlatforms", testPolicyQueriesForHostPlatforms}, {"PoliciesByID", testPoliciesByID}, - {"PoliciesByName", testPoliciesByName}, {"TeamPolicyTransfer", testTeamPolicyTransfer}, {"ApplyPolicySpec", testApplyPolicySpec}, {"Save", testPoliciesSave}, @@ -583,10 +582,11 @@ func testTeamPolicyProprietary(t *testing.T, ds *Datastore) { require.Error(t, err) p, err := ds.NewTeamPolicy(ctx, team1.ID, &user1.ID, fleet.PolicyPayload{ - Name: "query1", - Query: "select 1;", - Description: "query1 desc", - Resolution: "query1 resolution", + Name: "query1", + Query: "select 1;", + Description: "query1 desc", + Resolution: "query1 resolution", + CalendarEventsEnabled: true, }) require.NoError(t, err) @@ -616,6 +616,7 @@ func testTeamPolicyProprietary(t *testing.T, ds *Datastore) { assert.Equal(t, "query1 resolution", *p.Resolution) require.NotNil(t, p.AuthorID) assert.Equal(t, user1.ID, *p.AuthorID) + assert.True(t, p.CalendarEventsEnabled) globalPolicies, err := ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) require.NoError(t, err) @@ -1115,47 +1116,6 @@ func testPoliciesByID(t *testing.T, ds *Datastore) { require.ErrorAs(t, err, &nfe) } -func testPoliciesByName(t *testing.T, ds *Datastore) { - ctx := context.Background() - user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) - policyName1 := "policy1" - policyName2 := "policy2" - _ = newTestPolicy(t, ds, user1, policyName1, "darwin", nil) - _ = newTestPolicy(t, ds, user1, policyName2, "darwin", nil) - team1, err := ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team1"}) - require.NoError(t, err) - - // No names provided - _, err = ds.PoliciesByName(context.Background(), []string{}, team1.ID) - require.Error(t, err) - - // Policies don't belong to a team - _, err = ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) - require.Error(t, err) - var nfe fleet.NotFoundError - require.ErrorAs(t, err, &nfe) - - policy1 := newTestPolicy(t, ds, user1, policyName1, "darwin", &team1.ID) - policy2 := newTestPolicy(t, ds, user1, policyName2, "darwin", &team1.ID) - - policiesByName, err := ds.PoliciesByName(context.Background(), []string{policyName1, policyName2}, team1.ID) - require.NoError(t, err) - require.Len(t, policiesByName, 2) - assert.Equal(t, policiesByName[policyName1].ID, policy1.ID) - assert.Equal(t, policiesByName[policyName2].ID, policy2.ID) - assert.Equal(t, policiesByName[policyName1].Name, policy1.Name) - assert.Equal(t, policiesByName[policyName2].Name, policy2.Name) - - // Policy does not exist - _, err = ds.PoliciesByName(context.Background(), []string{"doesn't exist"}, team1.ID) - assert.ErrorAs(t, err, &nfe) - - // One exists and one doesn't - _, err = ds.PoliciesByName(context.Background(), []string{policyName1, "doesn't exist"}, team1.ID) - assert.ErrorAs(t, err, &nfe) - -} - func testTeamPolicyTransfer(t *testing.T, ds *Datastore) { ctx := context.Background() user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) @@ -1286,12 +1246,13 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { Platform: "", }, { - Name: "query2", - Query: "select 2;", - Description: "query2 desc", - Resolution: "some other resolution", - Team: "team1", - Platform: "darwin", + Name: "query2", + Query: "select 2;", + Description: "query2 desc", + Resolution: "some other resolution", + Team: "team1", + Platform: "darwin", + CalendarEventsEnabled: true, }, { Name: "query3", @@ -1326,6 +1287,7 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { require.NotNil(t, teamPolicies[0].Resolution) assert.Equal(t, "some other resolution", *teamPolicies[0].Resolution) assert.Equal(t, "darwin", teamPolicies[0].Platform) + assert.True(t, teamPolicies[0].CalendarEventsEnabled) assert.Equal(t, "query3", teamPolicies[1].Name) assert.Equal(t, "select 3;", teamPolicies[1].Query) @@ -1335,6 +1297,7 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { require.NotNil(t, teamPolicies[1].Resolution) assert.Equal(t, "some other good resolution", *teamPolicies[1].Resolution) assert.Equal(t, "windows,linux", teamPolicies[1].Platform) + assert.False(t, teamPolicies[1].CalendarEventsEnabled) // Make sure apply is idempotent require.NoError(t, ds.ApplyPolicySpecs(ctx, user1.ID, []*fleet.PolicySpec{ @@ -1347,12 +1310,13 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { Platform: "", }, { - Name: "query2", - Query: "select 2;", - Description: "query2 desc", - Resolution: "some other resolution", - Team: "team1", - Platform: "darwin", + Name: "query2", + Query: "select 2;", + Description: "query2 desc", + Resolution: "some other resolution", + Team: "team1", + Platform: "darwin", + CalendarEventsEnabled: true, }, { Name: "query3", @@ -1382,12 +1346,13 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { Platform: "", }, { - Name: "query2", - Query: "select 2 from updated;", - Description: "query2 desc updated", - Resolution: "some other resolution updated", - Team: "team1", // No error, team did not change - Platform: "windows", + Name: "query2", + Query: "select 2 from updated;", + Description: "query2 desc updated", + Resolution: "some other resolution updated", + Team: "team1", // No error, team did not change + Platform: "windows", + CalendarEventsEnabled: false, }, })) policies, err = ds.ListGlobalPolicies(ctx, fleet.ListOptions{}) @@ -1402,6 +1367,7 @@ func testApplyPolicySpec(t *testing.T, ds *Datastore) { require.NotNil(t, policies[0].Resolution) assert.Equal(t, "some resolution updated", *policies[0].Resolution) assert.Equal(t, "", policies[0].Platform) + assert.False(t, policies[0].CalendarEventsEnabled) teamPolicies, _, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{}) require.NoError(t, err) @@ -1481,11 +1447,12 @@ func testPoliciesSave(t *testing.T, ds *Datastore) { assert.Equal(t, computeChecksum(*gp), hex.EncodeToString(globalChecksum)) payload = fleet.PolicyPayload{ - Name: "team1 query", - Query: "select 2;", - Description: "team1 query desc", - Resolution: "team1 query resolution", - Critical: true, + Name: "team1 query", + Query: "select 2;", + Description: "team1 query desc", + Resolution: "team1 query resolution", + Critical: true, + CalendarEventsEnabled: true, } tp1, err := ds.NewTeamPolicy(ctx, team1.ID, &user1.ID, payload) require.NoError(t, err) @@ -1494,6 +1461,7 @@ func testPoliciesSave(t *testing.T, ds *Datastore) { require.Equal(t, tp1.Description, payload.Description) require.Equal(t, *tp1.Resolution, payload.Resolution) require.Equal(t, tp1.Critical, payload.Critical) + assert.Equal(t, tp1.CalendarEventsEnabled, payload.CalendarEventsEnabled) var teamChecksum []uint8 err = ds.writer(context.Background()).Get(&teamChecksum, `SELECT checksum FROM policies WHERE id = ?`, tp1.ID) require.NoError(t, err) @@ -1522,6 +1490,7 @@ func testPoliciesSave(t *testing.T, ds *Datastore) { tp2.Description = "team1 query desc updated" tp2.Resolution = ptr.String("team1 query resolution updated") tp2.Critical = false + tp2.CalendarEventsEnabled = false err = ds.SavePolicy(ctx, &tp2, true) require.NoError(t, err) tp1, err = ds.Policy(ctx, tp1.ID) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 69af794e7..4081db8af 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -588,7 +588,6 @@ type Datastore interface { ListGlobalPolicies(ctx context.Context, opts ListOptions) ([]*Policy, error) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*Policy, error) - PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*Policy, error) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) CountPolicies(ctx context.Context, teamID *uint, matchQuery string) (int, error) UpdateHostPolicyCounts(ctx context.Context) error diff --git a/server/fleet/integrations.go b/server/fleet/integrations.go index 5f3c5e440..8c3450948 100644 --- a/server/fleet/integrations.go +++ b/server/fleet/integrations.go @@ -112,14 +112,9 @@ func (z TeamZendeskIntegration) UniqueKey() string { } type TeamGoogleCalendarIntegration struct { - Email string `json:"email"` - Enable bool `json:"enable_calendar_events"` - Policies []*PolicyRef `json:"policies"` - WebhookURL string `json:"webhook_url"` -} -type PolicyRef struct { - Name string `json:"name"` - ID uint `json:"id"` + Email string `json:"email"` + Enable bool `json:"enable_calendar_events"` + WebhookURL string `json:"webhook_url"` } // JiraIntegration configures an instance of an integration with the Jira @@ -380,7 +375,7 @@ func ValidateEnabledHostStatusIntegrations(webhook HostStatusWebhookSettings, in func ValidateGoogleCalendarIntegrations(intgs []*GoogleCalendarIntegration, invalid *InvalidArgumentError) { if len(intgs) > 1 { - invalid.Append("integrations.google_calendar", "only one Google Calendar integration is allowed at this time") + invalid.Append("integrations.google_calendar", "integrating with >1 Google Workspace service account is not yet supported.") } for _, intg := range intgs { intg.Email = strings.TrimSpace(intg.Email) diff --git a/server/fleet/policies.go b/server/fleet/policies.go index 78d57f86c..52a6109b2 100644 --- a/server/fleet/policies.go +++ b/server/fleet/policies.go @@ -30,6 +30,8 @@ type PolicyPayload struct { // // Empty string targets all platforms. Platform string + // CalendarEventsEnabled indicates whether calendar events are enabled for the policy. Only applies to team policies. + CalendarEventsEnabled bool } var ( @@ -107,6 +109,8 @@ type ModifyPolicyPayload struct { Platform *string `json:"platform"` // Critical marks the policy as high impact. Critical *bool `json:"critical" premium:"true"` + // CalendarEventsEnabled indicates whether calendar events are enabled for the policy. Only applies to team policies. + CalendarEventsEnabled *bool `json:"calendar_events_enabled" premium:"true"` } // Verify verifies the policy payload is valid. @@ -159,6 +163,8 @@ type PolicyData struct { // Empty string targets all platforms. Platform string `json:"platform" db:"platforms"` + CalendarEventsEnabled bool `json:"calendar_events_enabled" db:"calendar_events_enabled"` + UpdateCreateTimestamps } @@ -212,6 +218,8 @@ type PolicySpec struct { // // Empty string targets all platforms. Platform string `json:"platform,omitempty"` + // CalendarEventsEnabled indicates whether calendar events are enabled for the policy. Only applies to team policies. + CalendarEventsEnabled bool `json:"calendar_events_enabled"` } // Verify verifies the policy data is valid. diff --git a/server/fleet/teams.go b/server/fleet/teams.go index aee5c7ab9..9c9acbf4c 100644 --- a/server/fleet/teams.go +++ b/server/fleet/teams.go @@ -451,6 +451,11 @@ func TeamSpecFromTeam(t *Team) (*TeamSpec, error) { webhookSettings.HostStatusWebhook = t.Config.WebhookSettings.HostStatusWebhook } + var integrations TeamSpecIntegrations + if t.Config.Integrations.GoogleCalendar != nil { + integrations.GoogleCalendar = t.Config.Integrations.GoogleCalendar + } + return &TeamSpec{ Name: t.Name, AgentOptions: agentOptions, @@ -459,5 +464,6 @@ func TeamSpecFromTeam(t *Team) (*TeamSpec, error) { MDM: mdmSpec, HostExpirySettings: &t.Config.HostExpirySettings, WebhookSettings: webhookSettings, + Integrations: integrations, }, nil } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 603d0616f..146982697 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -432,8 +432,6 @@ type ListGlobalPoliciesFunc func(ctx context.Context, opts fleet.ListOptions) ([ type PoliciesByIDFunc func(ctx context.Context, ids []uint) (map[uint]*fleet.Policy, error) -type PoliciesByNameFunc func(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) - type DeleteGlobalPoliciesFunc func(ctx context.Context, ids []uint) ([]uint, error) type CountPoliciesFunc func(ctx context.Context, teamID *uint, matchQuery string) (int, error) @@ -1482,9 +1480,6 @@ type DataStore struct { PoliciesByIDFunc PoliciesByIDFunc PoliciesByIDFuncInvoked bool - PoliciesByNameFunc PoliciesByNameFunc - PoliciesByNameFuncInvoked bool - DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFunc DeleteGlobalPoliciesFuncInvoked bool @@ -3576,13 +3571,6 @@ func (s *DataStore) PoliciesByID(ctx context.Context, ids []uint) (map[uint]*fle return s.PoliciesByIDFunc(ctx, ids) } -func (s *DataStore) PoliciesByName(ctx context.Context, names []string, teamID uint) (map[string]*fleet.Policy, error) { - s.mu.Lock() - s.PoliciesByNameFuncInvoked = true - s.mu.Unlock() - return s.PoliciesByNameFunc(ctx, names, teamID) -} - func (s *DataStore) DeleteGlobalPolicies(ctx context.Context, ids []uint) ([]uint, error) { s.mu.Lock() s.DeleteGlobalPoliciesFuncInvoked = true diff --git a/server/service/client.go b/server/service/client.go index b1eb35d69..8384e3e54 100644 --- a/server/service/client.go +++ b/server/service/client.go @@ -880,7 +880,6 @@ func (c *Client) DoGitOps( } var mdmAppConfig map[string]interface{} var team map[string]interface{} - var teamCalendarIntegration map[string]interface{} if config.TeamName == nil { group.AppConfig = config.OrgSettings group.EnrollSecret = &fleet.EnrollSecretSpec{Secrets: config.OrgSettings["secrets"].([]*fleet.EnrollSecret)} @@ -968,20 +967,14 @@ func (c *Client) DoGitOps( if !ok { return errors.New("team_settings.integrations config is not a map") } - if calendar, ok := integrations.(map[string]interface{})["google_calendar"]; ok { - if calendar == nil { - calendar = map[string]interface{}{} - integrations.(map[string]interface{})["google_calendar"] = calendar - } - teamCalendarIntegration, ok = calendar.(map[string]interface{}) + if googleCal, ok := integrations.(map[string]interface{})["google_calendar"]; !ok || googleCal == nil { + integrations.(map[string]interface{})["google_calendar"] = map[string]interface{}{} + } else { + _, ok = googleCal.(map[string]interface{}) if !ok { return errors.New("team_settings.integrations.google_calendar config is not a map") } } - // We clear the calendar integration and re-apply it after updating policies. - // This is needed because the calendar integration may be referencing policies that need to be - // created/updated. - integrations.(map[string]interface{})["google_calendar"] = map[string]interface{}{} team["mdm"] = map[string]interface{}{} mdmAppConfig = team["mdm"].(map[string]interface{}) @@ -1087,23 +1080,6 @@ func (c *Client) DoGitOps( return err } - // Apply calendar integration - if len(teamCalendarIntegration) > 0 { - group = spec.Group{} - team = make(map[string]interface{}) - team["name"] = *config.TeamName - team["integrations"] = map[string]interface{}{"google_calendar": teamCalendarIntegration} - rawTeam, err := json.Marshal(team) - if err != nil { - return fmt.Errorf("error marshalling team spec: %w", err) - } - group.Teams = []json.RawMessage{rawTeam} - _, err = c.ApplyGroup(ctx, &group, baseDir, logf, fleet.ApplySpecOptions{DryRun: dryRun}) - if err != nil { - return err - } - } - err = c.doGitOpsQueries(config, logFn, dryRun) if err != nil { return err diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 2484ce0d1..22c2779ca 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -199,12 +199,7 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { "google_calendar": map[string]any{ "email": calendarEmail, "enable_calendar_events": true, - "policies": []any{ - map[string]any{ - "name": teamPolicy.Name, - }, - }, - "webhook_url": calendarWebhookUrl, + "webhook_url": calendarWebhookUrl, }, }, }, @@ -218,8 +213,6 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { assert.Equal(t, calendarEmail, team.Config.Integrations.GoogleCalendar.Email) assert.Equal(t, calendarWebhookUrl, team.Config.Integrations.GoogleCalendar.WebhookURL) assert.True(t, team.Config.Integrations.GoogleCalendar.Enable) - require.Len(t, team.Config.Integrations.GoogleCalendar.Policies, 1) - assert.Equal(t, teamPolicy.ID, team.Config.Integrations.GoogleCalendar.Policies[0].ID) // dry-run with invalid windows updates teamSpecs = map[string]any{ @@ -3686,7 +3679,7 @@ func (s *integrationEnterpriseTestSuite) TestGlobalPolicyCreateReadPatch() { } func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { - fields := []string{"Query", "Name", "Description", "Resolution", "Platform", "Critical"} + fields := []string{"Query", "Name", "Description", "Resolution", "Platform", "Critical", "CalendarEventsEnabled"} team1, err := s.ds.NewTeam(context.Background(), &fleet.Team{ ID: 42, @@ -3697,24 +3690,26 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { createPol1 := &teamPolicyResponse{} createPol1Req := &teamPolicyRequest{ - Query: "query", - Name: "name1", - Description: "description", - Resolution: "resolution", - Platform: "linux", - Critical: true, + Query: "query", + Name: "name1", + Description: "description", + Resolution: "resolution", + Platform: "linux", + Critical: true, + CalendarEventsEnabled: true, } s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team1.ID), createPol1Req, http.StatusOK, &createPol1) allEqual(s.T(), createPol1Req, createPol1.Policy, fields...) createPol2 := &teamPolicyResponse{} createPol2Req := &teamPolicyRequest{ - Query: "query", - Name: "name2", - Description: "description", - Resolution: "resolution", - Platform: "linux", - Critical: false, + Query: "query", + Name: "name2", + Description: "description", + Resolution: "resolution", + Platform: "linux", + Critical: false, + CalendarEventsEnabled: false, } s.DoJSON("POST", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team1.ID), createPol2Req, http.StatusOK, &createPol2) allEqual(s.T(), createPol2Req, createPol2.Policy, fields...) @@ -3730,12 +3725,13 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { patchPol1Req := &modifyTeamPolicyRequest{ ModifyPolicyPayload: fleet.ModifyPolicyPayload{ - Name: ptr.String("newName1"), - Query: ptr.String("newQuery"), - Description: ptr.String("newDescription"), - Resolution: ptr.String("newResolution"), - Platform: ptr.String("windows"), - Critical: ptr.Bool(false), + Name: ptr.String("newName1"), + Query: ptr.String("newQuery"), + Description: ptr.String("newDescription"), + Resolution: ptr.String("newResolution"), + Platform: ptr.String("windows"), + Critical: ptr.Bool(false), + CalendarEventsEnabled: ptr.Bool(false), }, } patchPol1 := &modifyTeamPolicyResponse{} @@ -3744,12 +3740,13 @@ func (s *integrationEnterpriseTestSuite) TestTeamPolicyCreateReadPatch() { patchPol2Req := &modifyTeamPolicyRequest{ ModifyPolicyPayload: fleet.ModifyPolicyPayload{ - Name: ptr.String("newName2"), - Query: ptr.String("newQuery"), - Description: ptr.String("newDescription"), - Resolution: ptr.String("newResolution"), - Platform: ptr.String("windows"), - Critical: ptr.Bool(true), + Name: ptr.String("newName2"), + Query: ptr.String("newQuery"), + Description: ptr.String("newDescription"), + Resolution: ptr.String("newResolution"), + Platform: ptr.String("windows"), + Critical: ptr.Bool(true), + CalendarEventsEnabled: ptr.Bool(true), }, } patchPol2 := &modifyTeamPolicyResponse{} diff --git a/server/service/team_policies.go b/server/service/team_policies.go index 40187769b..15a7a90ac 100644 --- a/server/service/team_policies.go +++ b/server/service/team_policies.go @@ -20,14 +20,15 @@ import ( ///////////////////////////////////////////////////////////////////////////////// type teamPolicyRequest struct { - TeamID uint `url:"team_id"` - QueryID *uint `json:"query_id"` - Query string `json:"query"` - Name string `json:"name"` - Description string `json:"description"` - Resolution string `json:"resolution"` - Platform string `json:"platform"` - Critical bool `json:"critical" premium:"true"` + TeamID uint `url:"team_id"` + QueryID *uint `json:"query_id"` + Query string `json:"query"` + Name string `json:"name"` + Description string `json:"description"` + Resolution string `json:"resolution"` + Platform string `json:"platform"` + Critical bool `json:"critical" premium:"true"` + CalendarEventsEnabled bool `json:"calendar_events_enabled"` } type teamPolicyResponse struct { @@ -40,13 +41,14 @@ func (r teamPolicyResponse) error() error { return r.Err } func teamPolicyEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { req := request.(*teamPolicyRequest) resp, err := svc.NewTeamPolicy(ctx, req.TeamID, fleet.PolicyPayload{ - QueryID: req.QueryID, - Name: req.Name, - Query: req.Query, - Description: req.Description, - Resolution: req.Resolution, - Platform: req.Platform, - Critical: req.Critical, + QueryID: req.QueryID, + Name: req.Name, + Query: req.Query, + Description: req.Description, + Resolution: req.Resolution, + Platform: req.Platform, + Critical: req.Critical, + CalendarEventsEnabled: req.CalendarEventsEnabled, }) if err != nil { return teamPolicyResponse{Err: err}, nil @@ -390,6 +392,9 @@ func (svc *Service) modifyPolicy(ctx context.Context, teamID *uint, id uint, p f if p.Critical != nil { policy.Critical = *p.Critical } + if p.CalendarEventsEnabled != nil { + policy.CalendarEventsEnabled = *p.CalendarEventsEnabled + } logging.WithExtras(ctx, "name", policy.Name, "sql", policy.Query) err = svc.ds.SavePolicy(ctx, policy, shouldRemoveAll) diff --git a/server/webhooks/failing_policies_test.go b/server/webhooks/failing_policies_test.go index 71b890cfe..1c2edda53 100644 --- a/server/webhooks/failing_policies_test.go +++ b/server/webhooks/failing_policies_test.go @@ -124,7 +124,8 @@ func TestTriggerFailingPoliciesWebhookBasic(t *testing.T) { "passing_host_count": 0, "failing_host_count": 0, "host_count_updated_at": null, - "critical": true + "critical": true, + "calendar_events_enabled": false }, "hosts": [ { @@ -183,16 +184,17 @@ func TestTriggerFailingPoliciesWebhookTeam(t *testing.T) { policiesByID := map[uint]*fleet.Policy{ 1: { PolicyData: fleet.PolicyData{ - ID: 1, - Name: "policy1", - Query: "select 1", - Description: "policy1 description", - AuthorID: ptr.Uint(1), - AuthorName: "Alice", - AuthorEmail: "alice@example.com", - TeamID: &teamID, - Resolution: ptr.String("policy1 resolution"), - Platform: "darwin", + ID: 1, + Name: "policy1", + Query: "select 1", + Description: "policy1 description", + AuthorID: ptr.Uint(1), + AuthorName: "Alice", + AuthorEmail: "alice@example.com", + TeamID: &teamID, + Resolution: ptr.String("policy1 resolution"), + Platform: "darwin", + CalendarEventsEnabled: true, }, }, 2: { @@ -309,7 +311,8 @@ func TestTriggerFailingPoliciesWebhookTeam(t *testing.T) { "passing_host_count": 0, "failing_host_count": 0, "host_count_updated_at": null, - "critical": false + "critical": false, + "calendar_events_enabled": true }, "hosts": [ { From 2db8eb3c80ed1de49b91dc4b539926cf9b094431 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Thu, 14 Mar 2024 19:15:35 -0500 Subject: [PATCH 06/36] Update migrations for main rebase. --- ... 20240314085226_AddCalendarEventTables.go} | 6 ++-- ...0314085226_AddCalendarEventTables_test.go} | 2 +- server/datastore/mysql/schema.sql | 33 +++++++++++++++++-- 3 files changed, 35 insertions(+), 6 deletions(-) rename server/datastore/mysql/migrations/tables/{20240313085226_AddCalendarEventTables.go => 20240314085226_AddCalendarEventTables.go} (89%) rename server/datastore/mysql/migrations/tables/{20240313085226_AddCalendarEventTables_test.go => 20240314085226_AddCalendarEventTables_test.go} (97%) diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go similarity index 89% rename from server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go rename to server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go index 242a11ecb..e9222e9d9 100644 --- a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables.go +++ b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go @@ -6,10 +6,10 @@ import ( ) func init() { - MigrationClient.AddMigration(Up_20240313085226, Down_20240313085226) + MigrationClient.AddMigration(Up_20240314085226, Down_20240314085226) } -func Up_20240313085226(tx *sql.Tx) error { +func Up_20240314085226(tx *sql.Tx) error { // TODO(lucas): Check if we need more indexes. if _, err := tx.Exec(` @@ -47,6 +47,6 @@ func Up_20240313085226(tx *sql.Tx) error { return nil } -func Down_20240313085226(tx *sql.Tx) error { +func Down_20240314085226(tx *sql.Tx) error { return nil } diff --git a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables_test.go similarity index 97% rename from server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go rename to server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables_test.go index 216a3e468..85f61b342 100644 --- a/server/datastore/mysql/migrations/tables/20240313085226_AddCalendarEventTables_test.go +++ b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestUp_20240313085226(t *testing.T) { +func TestUp_20240314085226(t *testing.T) { db := applyUpToPrev(t) applyNext(t, db) diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 0040d4990..a256591e2 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -44,6 +44,19 @@ CREATE TABLE `app_config_json` ( INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"windows_updates\": {\"deadline_days\": null, \"grace_period_days\": null}, \"windows_settings\": {\"custom_settings\": null}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enable_disk_encryption\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"scripts\": null, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null, \"google_calendar\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"scripts_disabled\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `calendar_events` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `email` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + `start_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `end_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `event` json NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `carve_blocks` ( `metadata_id` int(10) unsigned NOT NULL, `block_id` int(11) NOT NULL, @@ -192,6 +205,21 @@ CREATE TABLE `host_batteries` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `host_calendar_events` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `host_id` int(10) unsigned NOT NULL, + `calendar_event_id` int(10) unsigned NOT NULL, + `webhook_status` tinyint(4) NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_one_calendar_event_per_host` (`host_id`), + KEY `calendar_event_id` (`calendar_event_id`), + CONSTRAINT `host_calendar_events_ibfk_1` FOREIGN KEY (`calendar_event_id`) REFERENCES `calendar_events` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `host_dep_assignments` ( `host_id` int(10) unsigned NOT NULL, `added_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -779,9 +807,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=257 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=259 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1098,6 +1126,7 @@ CREATE TABLE `policies` ( `platforms` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `critical` tinyint(1) NOT NULL DEFAULT '0', `checksum` binary(16) NOT NULL, + `calendar_events_enabled` tinyint(1) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `idx_policies_checksum` (`checksum`), KEY `idx_policies_author_id` (`author_id`), From 21f95d8b5d2f3530eb46e35b7d02ba36328f98e5 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 15 Mar 2024 10:26:58 -0500 Subject: [PATCH 07/36] Calendar interface fixes from code review and refactoring. (#17658) Calendar interface fixes from code review and manual merge with @lucasmrod changes. --- ee/server/calendar/calendar.go | 16 ----- ee/server/calendar/google_calendar.go | 86 ++++++++++++++------------- server/fleet/calendar.go | 11 ++-- 3 files changed, 50 insertions(+), 63 deletions(-) delete mode 100644 ee/server/calendar/calendar.go diff --git a/ee/server/calendar/calendar.go b/ee/server/calendar/calendar.go deleted file mode 100644 index 469437f3a..000000000 --- a/ee/server/calendar/calendar.go +++ /dev/null @@ -1,16 +0,0 @@ -package calendar - -import ( - "context" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/go-kit/kit/log" -) - -type GoogleCalendarConfig struct { - Context context.Context - IntegrationConfig *fleet.GoogleCalendarIntegration - UserEmail string - Logger log.Logger - // Should be nil for production - API GoogleCalendarAPI -} diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index ff3d2dfb8..ce1e9cfb9 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -5,16 +5,18 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "time" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" + kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" "golang.org/x/oauth2/google" "golang.org/x/oauth2/jwt" "google.golang.org/api/calendar/v3" "google.golang.org/api/googleapi" "google.golang.org/api/option" - "net/http" - "time" ) const ( @@ -30,15 +32,34 @@ var calendarScopes = []string{ "https://www.googleapis.com/auth/calendar.settings.readonly", } -// GoogleCalendar is an implementation of the Calendar interface that uses the +type GoogleCalendarConfig struct { + Context context.Context + IntegrationConfig *fleet.GoogleCalendarIntegration + Logger kitlog.Logger + // Should be nil for production + API GoogleCalendarAPI +} + +// GoogleCalendar is an implementation of the UserCalendar interface that uses the // Google Calendar API to manage events. type GoogleCalendar struct { - config *GoogleCalendarConfig - timezoneOffset *int + config *GoogleCalendarConfig + currentUserEmail string + timezoneOffset *int +} + +func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { + if config.API == nil { + var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} + config.API = lowLevelAPI + } + return &GoogleCalendar{ + config: config, + } } type GoogleCalendarAPI interface { - Connect(ctx context.Context, email, privateKey, subject string) error + Configure(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error GetSetting(name string) (*calendar.Setting, error) ListEvents(timeMin, timeMax string) (*calendar.Events, error) CreateEvent(event *calendar.Event) (*calendar.Event, error) @@ -55,15 +76,17 @@ type GoogleCalendarLowLevelAPI struct { service *calendar.Service } -// Connect creates a new Google Calendar service using the provided credentials. -func (lowLevelAPI *GoogleCalendarLowLevelAPI) Connect(ctx context.Context, email, privateKey, subject string) error { +// Configure creates a new Google Calendar service using the provided credentials. +func (lowLevelAPI *GoogleCalendarLowLevelAPI) Configure( + ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string, +) error { // Create a new calendar service conf := &jwt.Config{ - Email: email, + Email: serviceAccountEmail, Scopes: calendarScopes, PrivateKey: []byte(privateKey), TokenURL: google.JWTTokenURL, - Subject: subject, + Subject: userToImpersonateEmail, } client := conf.Client(ctx) service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) @@ -95,33 +118,18 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { return lowLevelAPI.service.Events.Delete(calendarID, id).Do() } -func (c *GoogleCalendar) Connect(config any) (fleet.Calendar, error) { - gConfig, ok := config.(*GoogleCalendarConfig) - if !ok { - return nil, errors.New("invalid Google calendar config") - } - if gConfig.API == nil { - var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} - gConfig.API = lowLevelAPI - } - err := gConfig.API.Connect( - gConfig.Context, gConfig.IntegrationConfig.Email, gConfig.IntegrationConfig.PrivateKey, gConfig.UserEmail, +func (c *GoogleCalendar) Configure(userEmail string) error { + err := c.config.API.Configure( + c.config.Context, c.config.IntegrationConfig.Email, c.config.IntegrationConfig.PrivateKey, userEmail, ) if err != nil { - return nil, ctxerr.Wrap(gConfig.Context, err, "creating Google calendar service") + return ctxerr.Wrap(c.config.Context, err, "creating Google calendar service") } - - gCal := &GoogleCalendar{ - config: gConfig, - } - - return gCal, nil + c.currentUserEmail = userEmail + return nil } func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { - if c.config == nil { - return nil, false, errors.New("the Google calendar is not connected. Please call Connect first") - } if event.EndTime.Before(time.Now()) { return nil, false, ctxerr.Errorf(c.config.Context, "cannot get and update an event that has already ended: %s", event.EndTime) } @@ -199,16 +207,11 @@ func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDet if details.ID == "" { return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ID") } - if details.ETag == "" { - return nil, ctxerr.Errorf(c.config.Context, "missing Google calendar event ETag") - } + // ETag is optional, but we need it to check if the event was modified return &details, nil } func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { - if c.config == nil { - return nil, errors.New("the Google calendar is not connected. Please call Connect first") - } if c.timezoneOffset == nil { err := getTimezone(c) if err != nil { @@ -257,7 +260,7 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. attending = true } else { for _, attendee := range gEvent.Attendees { - if attendee.Email == c.config.UserEmail { + if attendee.Email == c.currentUserEmail { if attendee.ResponseStatus != "declined" { attending = true } @@ -316,8 +319,7 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. if err != nil { return nil, err } - level.Debug(c.config.Logger).Log("msg", "created Google calendar events", "user", c.config.UserEmail, "startTime", eventStart) - fmt.Printf("VICTOR Created event with id:%s and ETag:%s\n", event.Id, event.Etag) + level.Debug(c.config.Logger).Log("msg", "created Google calendar event", "user", c.currentUserEmail, "startTime", eventStart) return fleetEvent, nil } @@ -364,7 +366,7 @@ func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime ti fleetEvent := &fleet.CalendarEvent{} fleetEvent.StartTime = startTime fleetEvent.EndTime = endTime - fleetEvent.Email = c.config.UserEmail + fleetEvent.Email = c.currentUserEmail details := &eventDetails{ ID: event.Id, ETag: event.Etag, @@ -379,7 +381,7 @@ func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime ti func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error { if c.config == nil { - return errors.New("the Google calendar is not connected. Please call Connect first") + return errors.New("the Google calendar is not connected. Please call Configure first") } details, err := c.unmarshalDetails(event) if err != nil { diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index dc682e24e..db2bbbc45 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -10,15 +10,16 @@ func (e DayEndedError) Error() string { return e.Msg } -type Calendar interface { - // Connect to calendar. This method must be called first. Currently, config must be a *GoogleCalendarConfig - Connect(config any) (Calendar, error) +type UserCalendar interface { + // Configure configures the connection to a user's calendar. Once configured, + // CreateEvent, GetAndUpdateEvent and DeleteEvent reference the user's calendar. + Configure(userEmail string) error + // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. + CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) // GetAndUpdateEvent retrieves the event from the calendar. // If the event has been modified, it returns the updated event. // If the event has been deleted, it schedules a new event with given body callback and returns the new event. GetAndUpdateEvent(event *CalendarEvent, genBodyFn func() string) (updatedEvent *CalendarEvent, updated bool, err error) - // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. - CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) // DeleteEvent deletes the event with the given ID. DeleteEvent(event *CalendarEvent) error } From 712d776be14d688742af4a7ea6f11abf21f7c27e Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 18 Mar 2024 14:44:07 -0500 Subject: [PATCH 08/36] Calendar interface (tests and associated fixes) (#17665) Completed unit tests for Google calendar interface, along with bug fixes. # Checklist for submitter - [ ] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --- ee/server/calendar/google_calendar.go | 174 ++++-- ee/server/calendar/google_calendar_test.go | 589 +++++++++++++++++++++ 2 files changed, 708 insertions(+), 55 deletions(-) create mode 100644 ee/server/calendar/google_calendar_test.go diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index ce1e9cfb9..53e189e24 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -111,7 +111,14 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calend func (lowLevelAPI *GoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { // Default maximum number of events returned is 250, which should be sufficient for most calendars. - return lowLevelAPI.service.Events.List(calendarID).EventTypes("default").OrderBy("startTime").SingleEvents(true).TimeMin(timeMin).TimeMax(timeMax).Do() + return lowLevelAPI.service.Events.List(calendarID). + EventTypes("default"). + OrderBy("startTime"). + SingleEvents(true). + TimeMin(timeMin). + TimeMax(timeMax). + ShowDeleted(false). + Do() } func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { @@ -130,9 +137,7 @@ func (c *GoogleCalendar) Configure(userEmail string) error { } func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { - if event.EndTime.Before(time.Now()) { - return nil, false, ctxerr.Errorf(c.config.Context, "cannot get and update an event that has already ended: %s", event.EndTime) - } + // We assume that the Fleet event has not already ended. We will simply return it if it has not been modified. details, err := c.unmarshalDetails(event) if err != nil { return nil, false, err @@ -143,6 +148,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn // http.StatusNotModified is returned sometimes, but not always, so we need to check ETag explicitly later case googleapi.IsNotModified(err): return event, false, nil + // http.StatusNotFound should be very rare -- Google keeps events for a while after they are deleted case isNotFound(err): deleted = true case err != nil: @@ -153,21 +159,50 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn // Event was not modified return event, false, nil } - endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) - if err != nil { - return nil, false, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), - ) + if gEvent.End == nil || (gEvent.End.DateTime == "" && gEvent.End.Date == "") { + // We should not see this error. If we do, we can work around by treating event as deleted. + return nil, false, ctxerr.Errorf(c.config.Context, "missing end date/time for Google calendar event: %s", gEvent.Id) } - // If event already ended, it is effectively deleted - if endTime.After(time.Now()) { - startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + + if gEvent.End.DateTime == "" { + // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. + // We won't handle all-day events at this time, and treat the event as deleted. + deleted = true + } + + var endTime *time.Time + if !deleted { + endTime, err = c.parseDateTime(gEvent.End) if err != nil { - return nil, false, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), - ) + return nil, false, err } - fleetEvent, err := c.googleEventToFleetEvent(startTime, endTime, gEvent) + if !endTime.After(time.Now()) { + // If event already ended, it is effectively deleted + // Delete this event to prevent confusion. This operation should be rare. + err = c.DeleteEvent(event) + if err != nil { + level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which is in the past", "err", err) + } + deleted = true + } + } + if !deleted { + if gEvent.Start == nil || (gEvent.Start.DateTime == "" && gEvent.Start.Date == "") { + // We should not see this error. If we do, we can work around by treating event as deleted. + return nil, false, ctxerr.Errorf(c.config.Context, "missing start date/time for Google calendar event: %s", gEvent.Id) + } + if gEvent.Start.DateTime == "" { + // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. + // We won't handle all-day events at this time, and treat the event as deleted. + deleted = true + } + } + if !deleted { + startTime, err := c.parseDateTime(gEvent.Start) + if err != nil { + return nil, false, err + } + fleetEvent, err := c.googleEventToFleetEvent(*startTime, *endTime, gEvent) if err != nil { return nil, false, err } @@ -175,12 +210,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn } } - newStartDate := event.StartTime.Add(24 * time.Hour) - if newStartDate.Weekday() == time.Saturday { - newStartDate = newStartDate.Add(48 * time.Hour) - } else if newStartDate.Weekday() == time.Sunday { - newStartDate = newStartDate.Add(24 * time.Hour) - } + newStartDate := calculateNewEventDate(event.StartTime) fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn()) if err != nil { @@ -189,6 +219,34 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn return fleetEvent, true, nil } +func calculateNewEventDate(oldStartDate time.Time) time.Time { + // Note: we do not handle time changes (daylight savings time, etc.) -- assuming 1 day is always 24 hours. + newStartDate := oldStartDate.Add(24 * time.Hour) + if newStartDate.Weekday() == time.Saturday { + newStartDate = newStartDate.Add(48 * time.Hour) + } else if newStartDate.Weekday() == time.Sunday { + newStartDate = newStartDate.Add(24 * time.Hour) + } + return newStartDate +} + +func (c *GoogleCalendar) parseDateTime(eventDateTime *calendar.EventDateTime) (*time.Time, error) { + var endTime time.Time + var err error + if eventDateTime.TimeZone != "" { + loc := getLocation(eventDateTime.TimeZone, c.config) + endTime, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc) + } else { + endTime, err = time.Parse(time.RFC3339, eventDateTime.DateTime) + } + if err != nil { + return nil, ctxerr.Wrap( + c.config.Context, err, fmt.Sprintf("parsing Google calendar event time: %s", eventDateTime.DateTime), + ) + } + return &endTime, nil +} + func isNotFound(err error) bool { if err == nil { return false @@ -212,6 +270,12 @@ func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDet } func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { + return c.createEvent(dayOfEvent, body, time.Now) +} + +// createEvent creates a new event on the calendar on the given date. timeNow is a function that returns the current time. +// timeNow can be overwritten for testing +func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow func() time.Time) (*fleet.CalendarEvent, error) { if c.timezoneOffset == nil { err := getTimezone(c) if err != nil { @@ -223,27 +287,26 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, location) dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, location) - now := time.Now().In(location) + now := timeNow().In(location) if dayEnd.Before(now) { // The workday has already ended. return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "cannot schedule an event for a day that has already ended"}) } // Adjust day start if workday already started - if dayStart.Before(now) { + if !dayStart.After(now) { dayStart = now.Truncate(eventLength) if dayStart.Before(now) { dayStart = dayStart.Add(eventLength) } - if dayStart.Equal(dayEnd) { + if !dayStart.Before(dayEnd) { return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "no time available for event"}) } } eventStart := dayStart eventEnd := dayStart.Add(eventLength) - searchStart := dayStart.Add(-24 * time.Hour) - events, err := c.config.API.ListEvents(searchStart.Format(time.RFC3339), dayEnd.Format(time.RFC3339)) + events, err := c.config.API.ListEvents(dayStart.Format(time.RFC3339), dayEnd.Format(time.RFC3339)) if err != nil { return nil, ctxerr.Wrap(c.config.Context, err, "listing Google calendar events") } @@ -253,48 +316,44 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet. continue } + // Ignore all day events + if gEvent.Start == nil || gEvent.Start.DateTime == "" || gEvent.End == nil || gEvent.End.DateTime == "" { + continue + } + // Ignore events that the user has declined - var attending bool - if len(gEvent.Attendees) == 0 { - // No attendees, so we assume the user is attending - attending = true - } else { - for _, attendee := range gEvent.Attendees { - if attendee.Email == c.currentUserEmail { - if attendee.ResponseStatus != "declined" { - attending = true - } + var declined bool + for _, attendee := range gEvent.Attendees { + if attendee.Email == c.currentUserEmail { + // The user has declined the event, so this time is open for scheduling + if attendee.ResponseStatus == "declined" { + declined = true break } } } - if !attending { + if declined { continue } // Ignore events that will end before our event - endTime, err := time.Parse(time.RFC3339, gEvent.End.DateTime) + endTime, err := c.parseDateTime(gEvent.End) if err != nil { - return nil, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event end time: %s", gEvent.End.DateTime), - ) + return nil, err } - if endTime.Before(eventStart) || endTime.Equal(eventStart) { + if !endTime.After(eventStart) { continue } - startTime, err := time.Parse(time.RFC3339, gEvent.Start.DateTime) + startTime, err := c.parseDateTime(gEvent.Start) if err != nil { - return nil, ctxerr.Wrap( - c.config.Context, err, fmt.Sprintf("parsing Google calendar event start time: %s", gEvent.Start.DateTime), - ) + return nil, err } if startTime.Before(eventEnd) { // Event occurs during our event, so we need to adjust. - fmt.Printf("VICTOR Adjusting event times due to %s: %s - %s\n", gEvent.Summary, eventStart, eventEnd) var isLastSlot bool - eventStart, eventEnd, isLastSlot = adjustEventTimes(endTime, dayEnd) + eventStart, eventEnd, isLastSlot = adjustEventTimes(*endTime, dayEnd) if isLastSlot { break } @@ -349,17 +408,22 @@ func getTimezone(gCal *GoogleCalendar) error { return ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone") } - loc, err := time.LoadLocation(setting.Value) - if err != nil { - // Could not load location, use EST - level.Warn(config.Logger).Log("msg", "parsing Google calendar timezone", "timezone", setting.Value, "err", err) - loc, _ = time.LoadLocation("America/New_York") - } + loc := getLocation(setting.Value, config) _, timezoneOffset := time.Now().In(loc).Zone() gCal.timezoneOffset = &timezoneOffset return nil } +func getLocation(name string, config *GoogleCalendarConfig) *time.Location { + loc, err := time.LoadLocation(name) + if err != nil { + // Could not load location, use EST + level.Warn(config.Logger).Log("msg", "parsing Google calendar timezone", "timezone", name, "err", err) + loc, _ = time.LoadLocation("America/New_York") + } + return loc +} + func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime time.Time, event *calendar.Event) ( *fleet.CalendarEvent, error, ) { diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go new file mode 100644 index 000000000..4c3e2db09 --- /dev/null +++ b/ee/server/calendar/google_calendar_test.go @@ -0,0 +1,589 @@ +package calendar + +import ( + "context" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" + "net/http" + "os" + "testing" + "time" +) + +const ( + baseServiceEmail = "service@example.com" + basePrivateKey = "private-key" + baseUserEmail = "user@example.com" +) + +var ( + baseCtx = context.Background() + logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) +) + +type MockGoogleCalendarLowLevelAPI struct { + ConfigureFunc func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error + GetSettingFunc func(name string) (*calendar.Setting, error) + ListEventsFunc func(timeMin, timeMax string) (*calendar.Events, error) + CreateEventFunc func(event *calendar.Event) (*calendar.Event, error) + GetEventFunc func(id, eTag string) (*calendar.Event, error) + DeleteEventFunc func(id string) error +} + +func (m *MockGoogleCalendarLowLevelAPI) Configure( + ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string, +) error { + return m.ConfigureFunc(ctx, serviceAccountEmail, privateKey, userToImpersonateEmail) +} + +func (m *MockGoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) { + return m.GetSettingFunc(name) +} + +func (m *MockGoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { + return m.ListEventsFunc(timeMin, timeMax) +} + +func (m *MockGoogleCalendarLowLevelAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + return m.CreateEventFunc(event) +} + +func (m *MockGoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calendar.Event, error) { + return m.GetEventFunc(id, eTag) +} + +func (m *MockGoogleCalendarLowLevelAPI) DeleteEvent(id string) error { + return m.DeleteEventFunc(id) +} + +func TestGoogleCalendar_Configure(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error { + assert.Equal(t, baseCtx, ctx) + assert.Equal(t, baseServiceEmail, serviceAccountEmail) + assert.Equal(t, basePrivateKey, privateKey) + assert.Equal(t, baseUserEmail, userToImpersonateEmail) + return nil + } + + // Happy path test + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + + // Configure error test + mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error { + return assert.AnError + } + err = cal.Configure(baseUserEmail) + assert.ErrorIs(t, err, assert.AnError) +} + +func makeConfig(mockAPI *MockGoogleCalendarLowLevelAPI) *GoogleCalendarConfig { + if mockAPI != nil && mockAPI.ConfigureFunc == nil { + mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error { + return nil + } + } + config := &GoogleCalendarConfig{ + Context: context.Background(), + IntegrationConfig: &fleet.GoogleCalendarIntegration{ + Email: baseServiceEmail, + PrivateKey: basePrivateKey, + }, + Logger: logger, + API: mockAPI, + } + return config +} + +func TestGoogleCalendar_DeleteEvent(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + mockAPI.DeleteEventFunc = func(id string) error { + assert.Equal(t, "event-id", id) + return nil + } + + // Happy path test + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)}) + assert.NoError(t, err) + + // API error test + mockAPI.DeleteEventFunc = func(id string) error { + return assert.AnError + } + err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)}) + assert.ErrorIs(t, err, assert.AnError) +} + +func TestGoogleCalendar_unmarshalDetails(t *testing.T) { + t.Parallel() + var gCal = NewGoogleCalendar(makeConfig(&MockGoogleCalendarLowLevelAPI{})) + err := gCal.Configure(baseUserEmail) + assert.NoError(t, err) + details, err := gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"id":"event-id","etag":"event-eTag"}`)}) + assert.NoError(t, err) + assert.Equal(t, "event-id", details.ID) + assert.Equal(t, "event-eTag", details.ETag) + + // Missing ETag is OK + details, err = gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"id":"event-id"}`)}) + assert.NoError(t, err) + assert.Equal(t, "event-id", details.ID) + assert.Equal(t, "", details.ETag) + + // Bad JSON + _, err = gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"bozo`)}) + assert.Error(t, err) + + // Missing id + _, err = gCal.unmarshalDetails(&fleet.CalendarEvent{Data: []byte(`{"myId":"event-id","etag":"event-eTag"}`)}) + assert.Error(t, err) +} + +func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + const baseETag = "event-eTag" + const baseEventID = "event-id" + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + assert.Equal(t, baseEventID, id) + assert.Equal(t, baseETag, eTag) + return &calendar.Event{ + Etag: baseETag, // ETag matches -- no modifications to event + }, nil + } + genBodyFn := func() string { + t.Error("genBodyFn should not be called") + return "event-body" + } + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + + eventStartTime := time.Now().UTC() + event := &fleet.CalendarEvent{ + StartTime: eventStartTime, + EndTime: time.Now().Add(time.Hour), + Data: []byte(`{"ID":"` + baseEventID + `","ETag":"` + baseETag + `"}`), + } + + // ETag matches + retrievedEvent, updated, err := cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.False(t, updated) + assert.Equal(t, event, retrievedEvent) + + // http.StatusNotModified response (ETag matches) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return nil, &googleapi.Error{Code: http.StatusNotModified} + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.False(t, updated) + assert.Equal(t, event, retrievedEvent) + + // Cannot unmarshal details + eventBadDetails := &fleet.CalendarEvent{ + StartTime: time.Now(), + EndTime: time.Now().Add(time.Hour), + Data: []byte(`{"bozo`), + } + _, _, err = cal.GetAndUpdateEvent(eventBadDetails, genBodyFn) + assert.Error(t, err) + + // API error test + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return nil, assert.AnError + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.ErrorIs(t, err, assert.AnError) + + // Event has been modified + startTime := time.Now().Add(time.Minute).Truncate(time.Second) + endTime := time.Now().Add(time.Hour).Truncate(time.Second) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + }, nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.True(t, updated) + assert.NotEqual(t, event, retrievedEvent) + require.NotNil(t, retrievedEvent) + assert.Equal(t, startTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, endTime.UTC(), retrievedEvent.EndTime.UTC()) + assert.Equal(t, baseUserEmail, retrievedEvent.Email) + gCal, _ := cal.(*GoogleCalendar) + details, err := gCal.unmarshalDetails(retrievedEvent) + require.NoError(t, err) + assert.Equal(t, "new-eTag", details.ETag) + assert.Equal(t, baseEventID, details.ID) + + // missing end time + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: ""}, + }, nil + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.Error(t, err) + + // missing start time + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + }, nil + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.Error(t, err) + + // Bad time format + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: "bozo"}, + }, nil + } + _, _, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.Error(t, err) + + // Event has been modified, with custom timezone. + tzId := "Africa/Kinshasa" + location, _ := time.LoadLocation(tzId) + startTime = time.Now().Add(time.Minute).Truncate(time.Second).In(location) + endTime = time.Now().Add(time.Hour).Truncate(time.Second).In(location) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.UTC().Format(time.RFC3339), TimeZone: tzId}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339), TimeZone: tzId}, + }, nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + assert.NoError(t, err) + assert.True(t, updated) + assert.NotEqual(t, event, retrievedEvent) + require.NotNil(t, retrievedEvent) + assert.Equal(t, startTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, endTime.UTC(), retrievedEvent.EndTime.UTC()) + assert.Equal(t, baseUserEmail, retrievedEvent.Email) + + // 404 response (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return nil, &googleapi.Error{Code: http.StatusNotFound} + } + mockAPI.GetSettingFunc = func(name string) (*calendar.Setting, error) { + return &calendar.Setting{Value: "UTC"}, nil + } + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return &calendar.Events{}, nil + } + genBodyFn = func() string { + return "event-body" + } + eventCreated := false + mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { + assert.Equal(t, eventTitle, event.Summary) + assert.Equal(t, genBodyFn(), event.Description) + event.Id = baseEventID + event.Etag = baseETag + eventCreated = true + return event, nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + assert.NotEqual(t, event, retrievedEvent) + require.NotNil(t, retrievedEvent) + assert.Equal(t, baseUserEmail, retrievedEvent.Email) + newEventDate := calculateNewEventDate(eventStartTime) + expectedStartTime := time.Date(newEventDate.Year(), newEventDate.Month(), newEventDate.Day(), startHour, 0, 0, 0, time.UTC) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) + + // cancelled (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + Status: "cancelled", + }, nil + } + eventCreated = false + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + require.NotNil(t, retrievedEvent) + assert.NotEqual(t, event, retrievedEvent) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) + + // all day event (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{Date: startTime.Format("2006-01-02")}, + End: &calendar.EventDateTime{DateTime: endTime.Format(time.RFC3339)}, + }, nil + } + eventCreated = false + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + require.NotNil(t, retrievedEvent) + assert.NotEqual(t, event, retrievedEvent) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) + + // moved in the past event (deleted) + mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { + return &calendar.Event{ + Id: baseEventID, + Etag: "new-eTag", + Start: &calendar.EventDateTime{DateTime: startTime.Add(-2 * time.Hour).Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: endTime.Add(-2 * time.Hour).Format(time.RFC3339)}, + }, nil + } + eventCreated = false + mockAPI.DeleteEventFunc = func(id string) error { + assert.Equal(t, baseEventID, id) + return nil + } + retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) + require.NoError(t, err) + assert.True(t, updated) + require.NotNil(t, retrievedEvent) + assert.NotEqual(t, event, retrievedEvent) + assert.Equal(t, expectedStartTime.UTC(), retrievedEvent.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), retrievedEvent.EndTime.UTC()) + assert.True(t, eventCreated) +} + +func TestGoogleCalendar_CreateEvent(t *testing.T) { + t.Parallel() + mockAPI := &MockGoogleCalendarLowLevelAPI{} + const baseEventID = "event-id" + const baseETag = "event-eTag" + const eventBody = "event-body" + var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI)) + err := cal.Configure(baseUserEmail) + assert.NoError(t, err) + + tzId := "Africa/Kinshasa" + mockAPI.GetSettingFunc = func(name string) (*calendar.Setting, error) { + return &calendar.Setting{Value: tzId}, nil + } + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return &calendar.Events{}, nil + } + mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { + assert.Equal(t, eventTitle, event.Summary) + assert.Equal(t, eventBody, event.Description) + event.Id = baseEventID + event.Etag = baseETag + return event, nil + } + + // Happy path test -- empty calendar + date := time.Now().Add(48 * time.Hour) + location, _ := time.LoadLocation(tzId) + expectedStartTime := time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + _, expectedOffset := expectedStartTime.Zone() + event, err := cal.CreateEvent(date, eventBody) + require.NoError(t, err) + assert.Equal(t, baseUserEmail, event.Email) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + _, offset := event.StartTime.Zone() + assert.Equal(t, expectedOffset, offset) + _, offset = event.EndTime.Zone() + assert.Equal(t, expectedOffset, offset) + gCal, _ := cal.(*GoogleCalendar) + details, err := gCal.unmarshalDetails(event) + require.NoError(t, err) + assert.Equal(t, baseETag, details.ETag) + assert.Equal(t, baseEventID, details.ID) + + // Workday already ended + date = time.Now().Add(-48 * time.Hour) + _, err = cal.CreateEvent(date, eventBody) + assert.ErrorAs(t, err, &fleet.DayEndedError{}) + + // There is no time left in the day to schedule an event + date = time.Now().Add(48 * time.Hour) + timeNow := func() time.Time { + now := time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 45, 0, 0, location) + return now + } + _, err = gCal.createEvent(date, eventBody, timeNow) + assert.ErrorAs(t, err, &fleet.DayEndedError{}) + + // Workday already started + date = time.Now().Add(48 * time.Hour) + expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) + timeNow = func() time.Time { + return expectedStartTime + } + event, err = gCal.createEvent(date, eventBody, timeNow) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // Busy calendar + date = time.Now().Add(48 * time.Hour) + dayStart := time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + dayEnd := time.Date(date.Year(), date.Month(), date.Day(), endHour, 0, 0, 0, location) + gEvents := &calendar.Events{} + // Cancelled event + gEvent := &calendar.Event{ + Id: "cancelled-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + Status: "cancelled", + } + gEvents.Items = append(gEvents.Items, gEvent) + // All day events + gEvent = &calendar.Event{ + Id: "all-day-event-id", + Start: &calendar.EventDateTime{Date: dayStart.Format(time.DateOnly)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + gEvent = &calendar.Event{ + Id: "all-day2-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{Date: dayEnd.Format(time.DateOnly)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + // User-declined event + gEvent = &calendar.Event{ + Id: "user-declined-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: baseUserEmail, ResponseStatus: "declined"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + // Event before day + gEvent = &calendar.Event{ + Id: "before-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Add(-time.Hour).Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayStart.Add(-30 * time.Minute).Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + + // Event from 6am to 11am + eventStart := time.Date(date.Year(), date.Month(), date.Day(), 6, 0, 0, 0, location) + eventEnd := time.Date(date.Year(), date.Month(), date.Day(), 11, 0, 0, 0, location) + gEvent = &calendar.Event{ + Id: "6-to-11-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: baseUserEmail, ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + + // Event from 10am to 10:30am + eventStart = time.Date(date.Year(), date.Month(), date.Day(), 10, 0, 0, 0, location) + eventEnd = time.Date(date.Year(), date.Month(), date.Day(), 10, 30, 0, 0, location) + gEvent = &calendar.Event{ + Id: "10-to-10-30-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: "other@example.com", ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + // Event from 11am to 11:45am + eventStart = time.Date(date.Year(), date.Month(), date.Day(), 11, 0, 0, 0, location) + eventEnd = time.Date(date.Year(), date.Month(), date.Day(), 11, 45, 0, 0, location) + gEvent = &calendar.Event{ + Id: "11-to-11-45-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: "other@example.com", ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + + // Event after day + eventStart = time.Date(date.Year(), date.Month(), date.Day(), endHour, 0, 0, 0, location) + eventEnd = time.Date(date.Year(), date.Month(), date.Day(), endHour, 45, 0, 0, location) + gEvent = &calendar.Event{ + Id: "after-event-id", + Start: &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)}, + Attendees: []*calendar.EventAttendee{{Email: "other@example.com", ResponseStatus: "accepted"}}, + } + gEvents.Items = append(gEvents.Items, gEvent) + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return gEvents, nil + } + expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), 12, 0, 0, 0, location) + event, err = gCal.CreateEvent(date, eventBody) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // Full schedule -- pick the last slot + date = time.Now().Add(48 * time.Hour) + dayStart = time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + dayEnd = time.Date(date.Year(), date.Month(), date.Day(), endHour, 0, 0, 0, location) + gEvents = &calendar.Events{} + gEvent = &calendar.Event{ + Id: "9-to-5-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return gEvents, nil + } + expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) + event, err = gCal.CreateEvent(date, eventBody) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // API error in ListEvents + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return nil, assert.AnError + } + _, err = gCal.CreateEvent(date, eventBody) + assert.ErrorIs(t, err, assert.AnError) + + // API error in CreateEvent + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return &calendar.Events{}, nil + } + mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { + return nil, assert.AnError + } + _, err = gCal.CreateEvent(date, eventBody) + assert.ErrorIs(t, err, assert.AnError) +} From 9a8ac02bc13d83ed93fb37ea2fffdc0f39289a77 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Tue, 19 Mar 2024 13:05:48 -0300 Subject: [PATCH 09/36] Happy path implementation of the calendar cron job (#17713) Happy path for #17441. --- cmd/fleet/calendar_cron.go | 454 ++++++++++++++++++ cmd/fleet/calendar_cron_test.go | 57 +++ cmd/fleet/serve.go | 12 + server/datastore/mysql/calendar_events.go | 150 ++++++ .../datastore/mysql/calendar_events_test.go | 6 + server/datastore/mysql/policies.go | 52 +- server/datastore/mysql/policies_test.go | 56 ++- server/fleet/app.go | 7 + server/fleet/calendar.go | 38 +- server/fleet/calendar_events.go | 9 + server/fleet/cron_schedules.go | 1 + server/fleet/datastore.go | 13 + server/fleet/policies.go | 5 + server/mock/datastore_mock.go | 96 ++++ server/service/osquery.go | 97 ++++ 15 files changed, 1050 insertions(+), 3 deletions(-) create mode 100644 cmd/fleet/calendar_cron.go create mode 100644 cmd/fleet/calendar_cron_test.go create mode 100644 server/datastore/mysql/calendar_events.go create mode 100644 server/datastore/mysql/calendar_events_test.go diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go new file mode 100644 index 000000000..099a938b2 --- /dev/null +++ b/cmd/fleet/calendar_cron.go @@ -0,0 +1,454 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/fleetdm/fleet/v4/ee/server/calendar" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/fleetdm/fleet/v4/server/service/schedule" + "github.com/go-kit/log" + kitlog "github.com/go-kit/log" + "github.com/go-kit/log/level" +) + +func newCalendarSchedule( + ctx context.Context, + instanceID string, + ds fleet.Datastore, + logger kitlog.Logger, +) (*schedule.Schedule, error) { + const ( + name = string(fleet.CronCalendar) + defaultInterval = 5 * time.Minute + ) + logger = kitlog.With(logger, "cron", name) + s := schedule.New( + ctx, name, instanceID, defaultInterval, ds, ds, + schedule.WithAltLockID("calendar"), + schedule.WithLogger(logger), + schedule.WithJob( + "calendar_events", + func(ctx context.Context) error { + return cronCalendarEvents(ctx, ds, logger) + }, + ), + ) + + return s, nil +} + +func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) error { + appConfig, err := ds.AppConfig(ctx) + if err != nil { + return fmt.Errorf("load app config: %w", err) + } + + if len(appConfig.Integrations.GoogleCalendar) == 0 { + return nil + } + googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0] + googleCalendarConfig := calendar.GoogleCalendarConfig{ + Context: ctx, + IntegrationConfig: googleCalendarIntegrationConfig, + Logger: log.With(logger, "component", "google_calendar"), + } + calendar := calendar.NewGoogleCalendar(&googleCalendarConfig) + domain := googleCalendarIntegrationConfig.Domain + + teams, err := ds.ListTeams(ctx, fleet.TeamFilter{ + User: &fleet.User{ + GlobalRole: ptr.String(fleet.RoleAdmin), + }, + }, fleet.ListOptions{}) + if err != nil { + return fmt.Errorf("list teams: %w", err) + } + + for _, team := range teams { + if err := cronCalendarEventsForTeam( + ctx, ds, calendar, *team, appConfig.OrgInfo.OrgName, domain, logger, + ); err != nil { + level.Info(logger).Log("msg", "events calendar cron", "team_id", team.ID, "err", err) + } + } + + return nil +} + +func cronCalendarEventsForTeam( + ctx context.Context, + ds fleet.Datastore, + calendar fleet.UserCalendar, + team fleet.Team, + orgName string, + domain string, + logger kitlog.Logger, +) error { + if team.Config.Integrations.GoogleCalendar == nil || + !team.Config.Integrations.GoogleCalendar.Enable { + return nil + } + + policies, err := ds.GetCalendarPolicies(ctx, team.ID) + if err != nil { + return fmt.Errorf("get calendar policy ids: %w", err) + } + + if len(policies) == 0 { + return nil + } + + logger = kitlog.With(logger, "team_id", team.ID) + + // + // NOTEs: + // - We ignore hosts that are passing all policies and do not have an associated email. + // - We get only one host per email that's failing policies (the one with lower host id). + // - On every host, we get only the first email that matches the domain (sorted lexicographically). + // + // TODOs(lucas): + // - We need to rate limit calendar requests. + // + + policyIDs := make([]uint, 0, len(policies)) + for _, policy := range policies { + policyIDs = append(policyIDs, policy.ID) + } + hosts, err := ds.GetHostsPolicyMemberships(ctx, domain, policyIDs) + if err != nil { + return fmt.Errorf("get team hosts failing policies: %w", err) + } + + var ( + passingHosts []fleet.HostPolicyMembershipData + failingHosts []fleet.HostPolicyMembershipData + failingHostsWithoutAssociatedEmail []fleet.HostPolicyMembershipData + ) + for _, host := range hosts { + if host.Passing { // host is passing all configured policies + if host.Email != "" { + passingHosts = append(passingHosts, host) + } + } else { // host is failing some of the configured policies + if host.Email == "" { + failingHostsWithoutAssociatedEmail = append(failingHostsWithoutAssociatedEmail, host) + } else { + failingHosts = append(failingHosts, host) + } + } + } + level.Debug(logger).Log( + "msg", "summary", + "passing_hosts", len(passingHosts), + "failing_hosts", len(failingHosts), + "failing_hosts_without_associated_email", len(failingHostsWithoutAssociatedEmail), + ) + + if err := processCalendarFailingHosts( + ctx, ds, calendar, orgName, failingHosts, logger, + ); err != nil { + level.Info(logger).Log("msg", "processing failing hosts", "err", err) + } + + // Remove calendar events from hosts that are passing the policies. + if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil { + level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err) + } + + // At last we want to notify the hosts that are failing and don't have an associated email. + if err := fireWebhookForHostsWithoutAssociatedEmail( + team.Config.Integrations.GoogleCalendar.WebhookURL, + domain, + failingHostsWithoutAssociatedEmail, + logger, + ); err != nil { + level.Info(logger).Log("msg", "webhook for hosts without associated email", "err", err) + } + + return nil +} + +func processCalendarFailingHosts( + ctx context.Context, + ds fleet.Datastore, + userCalendar fleet.UserCalendar, + orgName string, + hosts []fleet.HostPolicyMembershipData, + logger kitlog.Logger, +) error { + for _, host := range hosts { + logger := log.With(logger, "host_id", host.HostID) + if err := userCalendar.Configure(host.Email); err != nil { + return fmt.Errorf("configure user calendar: %w", err) + } + + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID) + + deletedExpiredEvent := false + if err == nil { + if calendarEvent.EndTime.Before(time.Now()) { + if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil { + level.Info(logger).Log("msg", "deleting existing expired calendar event", "err", err) + continue // continue with next host + } + deletedExpiredEvent = true + } + } + + switch { + case err == nil && !deletedExpiredEvent: + if err := processFailingHostExistingCalendarEvent( + ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err) + continue // continue with next host + } + case fleet.IsNotFound(err) || deletedExpiredEvent: + if err := processFailingHostCreateCalendarEvent( + ctx, ds, userCalendar, orgName, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host create calendar event", "err", err) + continue // continue with next host + } + default: + return fmt.Errorf("get calendar event: %w", err) + } + } + + return nil +} + +func processFailingHostExistingCalendarEvent( + ctx context.Context, + ds fleet.Datastore, + calendar fleet.UserCalendar, + orgName string, + hostCalendarEvent *fleet.HostCalendarEvent, + calendarEvent *fleet.CalendarEvent, + host fleet.HostPolicyMembershipData, +) error { + updatedEvent, updated, err := calendar.GetAndUpdateEvent(calendarEvent, func() string { + return generateCalendarEventBody(orgName, host.HostDisplayName) + }) + if err != nil { + return fmt.Errorf("get event calendar on db: %w", err) + } + if updated { + if err := ds.UpdateCalendarEvent(ctx, + calendarEvent.ID, + updatedEvent.StartTime, + updatedEvent.EndTime, + updatedEvent.Data, + ); err != nil { + return fmt.Errorf("updating event calendar on db: %w", err) + } + } + now := time.Now() + eventInFuture := now.Before(updatedEvent.StartTime) + if eventInFuture { + // If the webhook status was sent and event was moved to the future we set the status to pending. + // This can happen if the admin wants to retry a remediation. + if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent { + if err := ds.UpdateHostCalendarWebhookStatus(ctx, host.HostID, fleet.CalendarWebhookStatusPending); err != nil { + return fmt.Errorf("update host calendar webhook status: %w", err) + } + } + // Nothing else to do as event is in the future. + return nil + } + if now.After(updatedEvent.EndTime) { + return fmt.Errorf( + "unexpected event in the past: now=%s, start_time=%s, end_time=%s", + now, updatedEvent.StartTime, updatedEvent.EndTime, + ) + } + + // + // Event happening now. + // + + if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent { + return nil + } + + online, err := isHostOnline(ctx, ds, host.HostID) + if err != nil { + return fmt.Errorf("host online check: %w", err) + } + if !online { + // If host is offline then there's nothing to do. + return nil + } + + if err := ds.UpdateHostCalendarWebhookStatus(ctx, host.HostID, fleet.CalendarWebhookStatusPending); err != nil { + return fmt.Errorf("update host calendar webhook status: %w", err) + } + + // TODO(lucas): If this doesn't work at scale, then implement a special refetch + // for policies only. + if err := ds.UpdateHostRefetchRequested(ctx, host.HostID, true); err != nil { + return fmt.Errorf("refetch host: %w", err) + } + return nil +} + +func processFailingHostCreateCalendarEvent( + ctx context.Context, + ds fleet.Datastore, + userCalendar fleet.UserCalendar, + orgName string, + host fleet.HostPolicyMembershipData, +) error { + calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar) + if err != nil { + return fmt.Errorf("create event on user calendar: %w", err) + } + if _, err := ds.NewCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID); err != nil { + return fmt.Errorf("create calendar event on db: %w", err) + } + return nil +} + +func attemptCreatingEventOnUserCalendar( + orgName string, + host fleet.HostPolicyMembershipData, + userCalendar fleet.UserCalendar, +) (*fleet.CalendarEvent, error) { + // TODO(lucas): Where do we handle the following case (it seems CreateEvent needs to return no slot available for the requested day if there are none or too late): + // + // - If it’s the 3rd Tuesday of the month, create an event in the upcoming slot (if available). + // For example, if it’s the 3rd Tuesday of the month at 10:07a, Fleet will look for an open slot starting at 10:30a. + // - If it’s the 3rd Tuesday, Weds, Thurs, etc. of the month and it’s past the last slot, schedule the call for the next business day. + year, month, today := time.Now().Date() + preferredDate := getPreferredCalendarEventDate(year, month, today) + body := generateCalendarEventBody(orgName, host.HostDisplayName) + for { + calendarEvent, err := userCalendar.CreateEvent(preferredDate, body) + var dee fleet.DayEndedError + switch { + case err == nil: + return calendarEvent, nil + case errors.As(err, &dee): + preferredDate = addBusinessDay(preferredDate) + continue + default: + return nil, fmt.Errorf("create event on user calendar: %w", err) + } + } +} + +func getPreferredCalendarEventDate(year int, month time.Month, today int) time.Time { + const ( + // 3rd Tuesday of Month + preferredWeekDay = time.Tuesday + preferredOrdinal = 3 + ) + + firstDayOfMonth := time.Date(year, month, 1, 0, 0, 0, 0, time.UTC) + offset := int(preferredWeekDay - firstDayOfMonth.Weekday()) + if offset < 0 { + offset += 7 + } + preferredDate := firstDayOfMonth.AddDate(0, 0, offset+(7*(preferredOrdinal-1))) + if today > preferredDate.Day() { + today_ := time.Date(year, month, today, 0, 0, 0, 0, time.UTC) + preferredDate = addBusinessDay(today_) + } + return preferredDate +} + +func addBusinessDay(date time.Time) time.Time { + nextBusinessDay := 1 + switch weekday := date.Weekday(); weekday { + case time.Friday: + nextBusinessDay += 2 + case time.Saturday: + nextBusinessDay += 1 + } + return date.AddDate(0, 0, nextBusinessDay) +} + +func removeCalendarEventsFromPassingHosts( + ctx context.Context, + ds fleet.Datastore, + calendar fleet.UserCalendar, + hosts []fleet.HostPolicyMembershipData, +) error { + for _, host := range hosts { + calendarEvent, err := ds.GetCalendarEvent(ctx, host.Email) + switch { + case err == nil: + // OK + case fleet.IsNotFound(err): + continue + default: + return fmt.Errorf("get calendar event from DB: %w", err) + } + + if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil { + return fmt.Errorf("delete db calendar event: %w", err) + } + if err := calendar.Configure(host.Email); err != nil { + return fmt.Errorf("connect to user calendar: %w", err) + } + if err := calendar.DeleteEvent(calendarEvent); err != nil { + return fmt.Errorf("delete calendar event: %w", err) + } + } + return nil +} + +func fireWebhookForHostsWithoutAssociatedEmail( + webhookURL string, + domain string, + hosts []fleet.HostPolicyMembershipData, + logger kitlog.Logger, +) error { + // TODO(lucas): We are firing these every 5 minutes... + for _, host := range hosts { + if err := fleet.FireCalendarWebhook( + webhookURL, + host.HostID, host.HostHardwareSerial, host.HostDisplayName, nil, + fmt.Sprintf("No %s Google account associated with this host.", domain), + ); err != nil { + level.Error(logger).Log( + "msg", "fire webhook for hosts without associated email", "err", err, + ) + } + } + return nil +} + +func generateCalendarEventBody(orgName, hostDisplayName string) string { + return fmt.Sprintf(`Please leave your computer on and connected to power. + +Expect an automated restart. + +%s reserved this time to fix %s.`, orgName, hostDisplayName, + ) +} + +func isHostOnline(ctx context.Context, ds fleet.Datastore, hostID uint) (bool, error) { + hostLite, err := ds.HostLiteByID(ctx, hostID) + if err != nil { + return false, fmt.Errorf("get host lite: %w", err) + } + status := (&fleet.Host{ + DistributedInterval: hostLite.DistributedInterval, + ConfigTLSRefresh: hostLite.ConfigTLSRefresh, + SeenTime: hostLite.SeenTime, + }).Status(time.Now()) + + switch status { + case fleet.StatusOnline, fleet.StatusNew: + return true, nil + case fleet.StatusOffline, fleet.StatusMIA, fleet.StatusMissing: + return false, nil + default: + return false, fmt.Errorf("unknown host status: %s", status) + } +} diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go new file mode 100644 index 000000000..680cf50d9 --- /dev/null +++ b/cmd/fleet/calendar_cron_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestGetPreferredCalendarEventDate(t *testing.T) { + date := func(year int, month time.Month, day int) time.Time { + return time.Date(year, month, day, 0, 0, 0, 0, time.UTC) + } + for _, tc := range []struct { + name string + year int + month time.Month + days int + + expected time.Time + }{ + { + year: 2024, + month: 3, + days: 31, + name: "March 2024", + expected: date(2024, 3, 19), + }, + { + year: 2024, + month: 4, + days: 30, + name: "April 2024", + expected: date(2024, 4, 16), + }, + } { + t.Run(tc.name, func(t *testing.T) { + for day := 1; day <= tc.days; day++ { + actual := getPreferredCalendarEventDate(tc.year, tc.month, day) + require.NotEqual(t, actual.Weekday(), time.Saturday) + require.NotEqual(t, actual.Weekday(), time.Sunday) + if day <= tc.expected.Day() { + require.Equal(t, tc.expected, actual) + } else { + today := date(tc.year, tc.month, day) + if weekday := today.Weekday(); weekday == time.Friday { + require.Equal(t, today.AddDate(0, 0, +3), actual) + } else if weekday == time.Saturday { + require.Equal(t, today.AddDate(0, 0, +2), actual) + } else { + require.Equal(t, today.AddDate(0, 0, +1), actual) + } + } + } + }) + } +} diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 729cc3180..0971c3e39 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -768,6 +768,18 @@ the way that the Fleet server works. } } + if license.IsPremium() { + if err := cronSchedules.StartCronSchedule( + func() (fleet.CronSchedule, error) { + return newCalendarSchedule( + ctx, instanceID, ds, logger, + ) + }, + ); err != nil { + initFatal(err, "failed to register calendar schedule") + } + } + level.Info(logger).Log("msg", fmt.Sprintf("started cron schedules: %s", strings.Join(cronSchedules.ScheduleNames(), ", "))) // StartCollectors starts a goroutine per collector, using ctx to cancel. diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go new file mode 100644 index 000000000..399091597 --- /dev/null +++ b/server/datastore/mysql/calendar_events.go @@ -0,0 +1,150 @@ +package mysql + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/jmoiron/sqlx" +) + +func (ds *Datastore) NewCalendarEvent( + ctx context.Context, + email string, + startTime time.Time, + endTime time.Time, + data []byte, + hostID uint, +) (*fleet.CalendarEvent, error) { + var calendarEvent *fleet.CalendarEvent + if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + const calendarEventsQuery = ` + INSERT INTO calendar_events ( + email, + start_time, + end_time, + event + ) VALUES (?, ?, ?, ?); + ` + result, err := tx.ExecContext( + ctx, + calendarEventsQuery, + email, + startTime, + endTime, + data, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert calendar event") + } + + id, _ := result.LastInsertId() + calendarEvent = &fleet.CalendarEvent{ + ID: uint(id), + Email: email, + StartTime: startTime, + EndTime: endTime, + Data: data, + } + + const hostCalendarEventsQuery = ` + INSERT INTO host_calendar_events ( + host_id, + calendar_event_id, + webhook_status + ) VALUES (?, ?, ?); + ` + result, err = tx.ExecContext( + ctx, + hostCalendarEventsQuery, + hostID, + calendarEvent.ID, + fleet.CalendarWebhookStatusPending, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "insert host calendar event") + } + return nil + }); err != nil { + return nil, ctxerr.Wrap(ctx, err) + } + return calendarEvent, nil +} + +func (ds *Datastore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) { + const calendarEventsQuery = ` + SELECT * FROM calendar_events WHERE email = ?; + ` + var calendarEvent fleet.CalendarEvent + err := sqlx.GetContext(ctx, ds.reader(ctx), &calendarEvent, calendarEventsQuery, email) + if err != nil { + if err == sql.ErrNoRows { + return nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithMessage(fmt.Sprintf("email: %s", email))) + } + return nil, ctxerr.Wrap(ctx, err, "get calendar event") + } + return &calendarEvent, nil +} + +func (ds *Datastore) UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error { + const calendarEventsQuery = ` + UPDATE calendar_events SET + start_time = ?, + end_time = ?, + event = ? + WHERE id = ?; + ` + if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, startTime, endTime, data, calendarEventID); err != nil { + return ctxerr.Wrap(ctx, err, "update calendar event") + } + return nil +} + +func (ds *Datastore) DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error { + const calendarEventsQuery = ` + DELETE FROM calendar_events WHERE id = ?; + ` + if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, calendarEventID); err != nil { + return ctxerr.Wrap(ctx, err, "delete calendar event") + } + return nil +} + +func (ds *Datastore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + const hostCalendarEventsQuery = ` + SELECT * FROM host_calendar_events WHERE host_id = ? + ` + var hostCalendarEvent fleet.HostCalendarEvent + if err := sqlx.GetContext(ctx, ds.reader(ctx), &hostCalendarEvent, hostCalendarEventsQuery, hostID); err != nil { + if err == sql.ErrNoRows { + return nil, nil, ctxerr.Wrap(ctx, notFound("HostCalendarEvent").WithMessage(fmt.Sprintf("host_id: %d", hostID))) + } + return nil, nil, ctxerr.Wrap(ctx, err, "get host calendar event") + } + const calendarEventsQuery = ` + SELECT * FROM calendar_events WHERE id = ? + ` + var calendarEvent fleet.CalendarEvent + if err := sqlx.GetContext(ctx, ds.reader(ctx), &calendarEvent, calendarEventsQuery, hostCalendarEvent.CalendarEventID); err != nil { + if err == sql.ErrNoRows { + return nil, nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithID(hostCalendarEvent.CalendarEventID)) + } + return nil, nil, ctxerr.Wrap(ctx, err, "get calendar event") + } + return &hostCalendarEvent, &calendarEvent, nil +} + +func (ds *Datastore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error { + const calendarEventsQuery = ` + UPDATE host_calendar_events SET + webhook_status = ? + WHERE host_id = ?; + ` + if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, status, hostID); err != nil { + return ctxerr.Wrap(ctx, err, "update host calendar event webhook status") + } + return nil +} diff --git a/server/datastore/mysql/calendar_events_test.go b/server/datastore/mysql/calendar_events_test.go new file mode 100644 index 000000000..ccf07b3c7 --- /dev/null +++ b/server/datastore/mysql/calendar_events_test.go @@ -0,0 +1,6 @@ +package mysql + +import "testing" + +func TestCalendarEvents(t *testing.T) { +} diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index d2f242407..0b3498319 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -5,11 +5,12 @@ import ( "database/sql" "encoding/json" "fmt" - "golang.org/x/text/unicode/norm" "sort" "strings" "time" + "golang.org/x/text/unicode/norm" + "github.com/doug-martin/goqu/v9" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" @@ -1159,3 +1160,52 @@ func (ds *Datastore) UpdateHostPolicyCounts(ctx context.Context) error { return nil } + +func (ds *Datastore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + query := `SELECT id, name FROM policies WHERE team_id = ? AND calendar_events_enabled;` + var policies []fleet.PolicyCalendarData + err := sqlx.SelectContext(ctx, ds.reader(ctx), &policies, query, teamID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get calendar policies") + } + return policies, nil +} + +// TODO(lucas): Must be tested at scale. +func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) { + query := ` + SELECT + COALESCE(sh.email, '') AS email, + pm.passing AS passing, + h.id AS host_id, + hdn.display_name AS host_display_name, + h.hardware_serial AS host_hardware_serial + FROM ( + SELECT host_id, BIT_AND(COALESCE(passes, 0)) AS passing + FROM policy_membership + WHERE policy_id IN (?) + GROUP BY host_id + ) pm + LEFT JOIN ( + SELECT MIN(h.host_id) as host_id, h.email as email + FROM ( + SELECT host_id, MIN(email) AS email + FROM host_emails WHERE email LIKE CONCAT('%@', ?) + GROUP BY host_id + ) h GROUP BY h.email + ) sh ON sh.host_id = pm.host_id + JOIN hosts h ON h.id = pm.host_id + LEFT JOIN host_display_names hdn ON hdn.host_id = pm.host_id; +` + + query, args, err := sqlx.In(query, policyIDs, domain) + if err != nil { + return nil, ctxerr.Wrapf(ctx, err, "build select get team hosts policy memberships query") + } + var hosts []fleet.HostPolicyMembershipData + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &hosts, query, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "listing policies") + } + + return hosts, nil +} diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go index b0ef3b1bc..514de6dd3 100644 --- a/server/datastore/mysql/policies_test.go +++ b/server/datastore/mysql/policies_test.go @@ -59,6 +59,7 @@ func TestPolicies(t *testing.T) { {"TestPoliciesNameUnicode", testPoliciesNameUnicode}, {"TestPoliciesNameEmoji", testPoliciesNameEmoji}, {"TestPoliciesNameSort", testPoliciesNameSort}, + {"TestGetCalendarPolicies", testGetCalendarPolicies}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -2784,7 +2785,6 @@ func testPoliciesNameEmoji(t *testing.T, ds *Datastore) { assert.NoError(t, err) require.Len(t, policies, 1) assert.Equal(t, emoji1, policies[0].Name) - } // Ensure case-insensitive sort order for policy names @@ -2806,3 +2806,57 @@ func testPoliciesNameSort(t *testing.T, ds *Datastore) { assert.Equal(t, policy.Name, policiesResult[i].Name) } } + +func testGetCalendarPolicies(t *testing.T, ds *Datastore) { + ctx := context.Background() + + // Test with non-existent team. + _, err := ds.GetCalendarPolicies(ctx, 999) + require.NoError(t, err) + + team, err := ds.NewTeam(ctx, &fleet.Team{ + Name: "Foobar", + }) + require.NoError(t, err) + + // Test when the team has no policies. + _, err = ds.GetCalendarPolicies(ctx, team.ID) + require.NoError(t, err) + + // Create a global query to test that only team policies are returned. + _, err = ds.NewGlobalPolicy(ctx, nil, fleet.PolicyPayload{ + Name: "Global Policy", + Query: "SELECT * FROM time;", + }) + require.NoError(t, err) + + _, err = ds.NewTeamPolicy(ctx, team.ID, nil, fleet.PolicyPayload{ + Name: "Team Policy 1", + Query: "SELECT * FROM system_info;", + CalendarEventsEnabled: false, + }) + require.NoError(t, err) + + // Test when the team has policies, but none is configured for calendar. + _, err = ds.GetCalendarPolicies(ctx, team.ID) + require.NoError(t, err) + + teamPolicy2, err := ds.NewTeamPolicy(ctx, team.ID, nil, fleet.PolicyPayload{ + Name: "Team Policy 2", + Query: "SELECT * FROM osquery_info;", + CalendarEventsEnabled: true, + }) + require.NoError(t, err) + teamPolicy3, err := ds.NewTeamPolicy(ctx, team.ID, nil, fleet.PolicyPayload{ + Name: "Team Policy 3", + Query: "SELECT * FROM os_version;", + CalendarEventsEnabled: true, + }) + require.NoError(t, err) + + calendarPolicies, err := ds.GetCalendarPolicies(ctx, team.ID) + require.NoError(t, err) + require.Len(t, calendarPolicies, 2) + require.Equal(t, calendarPolicies[0].ID, teamPolicy2.ID) + require.Equal(t, calendarPolicies[1].ID, teamPolicy3.ID) +} diff --git a/server/fleet/app.go b/server/fleet/app.go index 4b936063a..560b8bb34 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -571,6 +571,13 @@ func (c *AppConfig) Copy() *AppConfig { clone.Integrations.Zendesk[i] = &zd } } + if len(c.Integrations.GoogleCalendar) > 0 { + clone.Integrations.GoogleCalendar = make([]*GoogleCalendarIntegration, len(c.Integrations.GoogleCalendar)) + for i, g := range c.Integrations.GoogleCalendar { + gc := *g + clone.Integrations.GoogleCalendar[i] = &gc + } + } if c.MDM.MacOSSettings.CustomSettings != nil { clone.MDM.MacOSSettings.CustomSettings = make([]MDMProfileSpec, len(c.MDM.MacOSSettings.CustomSettings)) diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index db2bbbc45..9b45c2c8a 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -1,6 +1,12 @@ package fleet -import "time" +import ( + "context" + "fmt" + "time" + + "github.com/fleetdm/fleet/v4/server" +) type DayEndedError struct { Msg string @@ -23,3 +29,33 @@ type UserCalendar interface { // DeleteEvent deletes the event with the given ID. DeleteEvent(event *CalendarEvent) error } + +type CalendarWebhookPayload struct { + Timestamp time.Time `json:"timestamp"` + HostID uint `json:"host_id"` + HostDisplayName string `json:"host_display_name"` + HostSerialNumber string `json:"host_serial_number"` + FailingPolicies []PolicyCalendarData `json:"failing_policies,omitempty"` + Error string `json:"error,omitempty"` +} + +func FireCalendarWebhook( + webhookURL string, + hostID uint, + hostHardwareSerial string, + hostDisplayName string, + failingCalendarPolicies []PolicyCalendarData, + err string, +) error { + if err := server.PostJSONWithTimeout(context.Background(), webhookURL, &CalendarWebhookPayload{ + Timestamp: time.Now(), + HostID: hostID, + HostDisplayName: hostDisplayName, + HostSerialNumber: hostHardwareSerial, + FailingPolicies: failingCalendarPolicies, + Error: err, + }); err != nil { + return fmt.Errorf("POST to %q: %w", server.MaskSecretURLParams(webhookURL), server.MaskURLError(err)) + } + return nil +} diff --git a/server/fleet/calendar_events.go b/server/fleet/calendar_events.go index 7671b4aba..348cb074a 100644 --- a/server/fleet/calendar_events.go +++ b/server/fleet/calendar_events.go @@ -27,3 +27,12 @@ type HostCalendarEvent struct { UpdateCreateTimestamps } + +type HostPolicyMembershipData struct { + Email string `db:"email"` + Passing bool `db:"passing"` + + HostID uint `db:"host_id"` + HostDisplayName string `db:"host_display_name"` + HostHardwareSerial string `db:"host_hardware_serial"` +} diff --git a/server/fleet/cron_schedules.go b/server/fleet/cron_schedules.go index 607f15f85..6b16734fd 100644 --- a/server/fleet/cron_schedules.go +++ b/server/fleet/cron_schedules.go @@ -21,6 +21,7 @@ const ( CronWorkerIntegrations CronScheduleName = "integrations" CronActivitiesStreaming CronScheduleName = "activities_streaming" CronMDMAppleProfileManager CronScheduleName = "mdm_apple_profile_manager" + CronCalendar CronScheduleName = "calendar" ) type CronSchedulesService interface { diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 4081db8af..f2178bf32 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -594,6 +594,9 @@ type Datastore interface { PolicyQueriesForHost(ctx context.Context, host *Host) (map[string]string, error) + GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]HostPolicyMembershipData, error) + GetCalendarPolicies(ctx context.Context, teamID uint) ([]PolicyCalendarData, error) + // Methods used for async processing of host policy query results. AsyncBatchInsertPolicyMembership(ctx context.Context, batch []PolicyMembershipResult) error AsyncBatchUpdatePolicyTimestamp(ctx context.Context, ids []uint, ts time.Time) error @@ -613,6 +616,16 @@ type Datastore interface { // the updated_at timestamp is older than the provided duration DeleteOutOfDateVulnerabilities(ctx context.Context, source VulnerabilitySource, duration time.Duration) error + /////////////////////////////////////////////////////////////////////////////// + // Calendar events + + NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*CalendarEvent, error) + GetCalendarEvent(ctx context.Context, email string) (*CalendarEvent, error) + DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error + UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error + GetHostCalendarEvent(ctx context.Context, hostID uint) (*HostCalendarEvent, *CalendarEvent, error) + UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status CalendarWebhookStatus) error + /////////////////////////////////////////////////////////////////////////////// // Team Policies diff --git a/server/fleet/policies.go b/server/fleet/policies.go index 52a6109b2..dda2ec047 100644 --- a/server/fleet/policies.go +++ b/server/fleet/policies.go @@ -179,6 +179,11 @@ type Policy struct { HostCountUpdatedAt *time.Time `json:"host_count_updated_at" db:"host_count_updated_at"` } +type PolicyCalendarData struct { + ID uint `db:"id" json:"id"` + Name string `db:"name" json:"name"` +} + func (p Policy) AuthzType() string { return "policy" } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 146982697..4e35d1eef 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -440,6 +440,10 @@ type UpdateHostPolicyCountsFunc func(ctx context.Context) error type PolicyQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error) +type GetHostsPolicyMembershipsFunc func(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) + +type GetCalendarPoliciesFunc func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) + type AsyncBatchInsertPolicyMembershipFunc func(ctx context.Context, batch []fleet.PolicyMembershipResult) error type AsyncBatchUpdatePolicyTimestampFunc func(ctx context.Context, ids []uint, ts time.Time) error @@ -458,6 +462,18 @@ type DeleteSoftwareVulnerabilitiesFunc func(ctx context.Context, vulnerabilities type DeleteOutOfDateVulnerabilitiesFunc func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error +type NewCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) + +type GetCalendarEventFunc func(ctx context.Context, email string) (*fleet.CalendarEvent, error) + +type DeleteCalendarEventFunc func(ctx context.Context, calendarEventID uint) error + +type UpdateCalendarEventFunc func(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error + +type GetHostCalendarEventFunc func(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) + +type UpdateHostCalendarWebhookStatusFunc func(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error + type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) type ListTeamPoliciesFunc func(ctx context.Context, teamID uint, opts fleet.ListOptions, iopts fleet.ListOptions) (teamPolicies []*fleet.Policy, inheritedPolicies []*fleet.Policy, err error) @@ -1492,6 +1508,12 @@ type DataStore struct { PolicyQueriesForHostFunc PolicyQueriesForHostFunc PolicyQueriesForHostFuncInvoked bool + GetHostsPolicyMembershipsFunc GetHostsPolicyMembershipsFunc + GetHostsPolicyMembershipsFuncInvoked bool + + GetCalendarPoliciesFunc GetCalendarPoliciesFunc + GetCalendarPoliciesFuncInvoked bool + AsyncBatchInsertPolicyMembershipFunc AsyncBatchInsertPolicyMembershipFunc AsyncBatchInsertPolicyMembershipFuncInvoked bool @@ -1519,6 +1541,24 @@ type DataStore struct { DeleteOutOfDateVulnerabilitiesFunc DeleteOutOfDateVulnerabilitiesFunc DeleteOutOfDateVulnerabilitiesFuncInvoked bool + NewCalendarEventFunc NewCalendarEventFunc + NewCalendarEventFuncInvoked bool + + GetCalendarEventFunc GetCalendarEventFunc + GetCalendarEventFuncInvoked bool + + DeleteCalendarEventFunc DeleteCalendarEventFunc + DeleteCalendarEventFuncInvoked bool + + UpdateCalendarEventFunc UpdateCalendarEventFunc + UpdateCalendarEventFuncInvoked bool + + GetHostCalendarEventFunc GetHostCalendarEventFunc + GetHostCalendarEventFuncInvoked bool + + UpdateHostCalendarWebhookStatusFunc UpdateHostCalendarWebhookStatusFunc + UpdateHostCalendarWebhookStatusFuncInvoked bool + NewTeamPolicyFunc NewTeamPolicyFunc NewTeamPolicyFuncInvoked bool @@ -3599,6 +3639,20 @@ func (s *DataStore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) return s.PolicyQueriesForHostFunc(ctx, host) } +func (s *DataStore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) { + s.mu.Lock() + s.GetHostsPolicyMembershipsFuncInvoked = true + s.mu.Unlock() + return s.GetHostsPolicyMembershipsFunc(ctx, domain, policyIDs) +} + +func (s *DataStore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + s.mu.Lock() + s.GetCalendarPoliciesFuncInvoked = true + s.mu.Unlock() + return s.GetCalendarPoliciesFunc(ctx, teamID) +} + func (s *DataStore) AsyncBatchInsertPolicyMembership(ctx context.Context, batch []fleet.PolicyMembershipResult) error { s.mu.Lock() s.AsyncBatchInsertPolicyMembershipFuncInvoked = true @@ -3662,6 +3716,48 @@ func (s *DataStore) DeleteOutOfDateVulnerabilities(ctx context.Context, source f return s.DeleteOutOfDateVulnerabilitiesFunc(ctx, source, duration) } +func (s *DataStore) NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) { + s.mu.Lock() + s.NewCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.NewCalendarEventFunc(ctx, email, startTime, endTime, data, hostID) +} + +func (s *DataStore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) { + s.mu.Lock() + s.GetCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.GetCalendarEventFunc(ctx, email) +} + +func (s *DataStore) DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error { + s.mu.Lock() + s.DeleteCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.DeleteCalendarEventFunc(ctx, calendarEventID) +} + +func (s *DataStore) UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error { + s.mu.Lock() + s.UpdateCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.UpdateCalendarEventFunc(ctx, calendarEventID, startTime, endTime, data) +} + +func (s *DataStore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + s.mu.Lock() + s.GetHostCalendarEventFuncInvoked = true + s.mu.Unlock() + return s.GetHostCalendarEventFunc(ctx, hostID) +} + +func (s *DataStore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error { + s.mu.Lock() + s.UpdateHostCalendarWebhookStatusFuncInvoked = true + s.mu.Unlock() + return s.UpdateHostCalendarWebhookStatusFunc(ctx, hostID, status) +} + func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { s.mu.Lock() s.NewTeamPolicyFuncInvoked = true diff --git a/server/service/osquery.go b/server/service/osquery.go index 8a77903a8..379afafd6 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -1001,6 +1001,10 @@ func (svc *Service) SubmitDistributedQueryResults( if len(policyResults) > 0 { + if err := processCalendarPolicies(ctx, svc.ds, ac, host, policyResults, svc.logger); err != nil { + logging.WithErr(ctx, err) + } + // filter policy results for webhooks var policyIDs []uint if globalPolicyAutomationsEnabled(ac.WebhookSettings, ac.Integrations) { @@ -1093,6 +1097,99 @@ func (svc *Service) SubmitDistributedQueryResults( return nil } +func processCalendarPolicies( + ctx context.Context, + ds fleet.Datastore, + appConfig *fleet.AppConfig, + host *fleet.Host, + policyResults map[uint]*bool, + logger log.Logger, +) error { + if len(appConfig.Integrations.GoogleCalendar) == 0 || host.TeamID == nil { + return nil + } + + team, err := ds.Team(ctx, *host.TeamID) + if err != nil { + return ctxerr.Wrap(ctx, err, "load host team") + } + + if team.Config.Integrations.GoogleCalendar == nil || !team.Config.Integrations.GoogleCalendar.Enable { + return nil + } + + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.ID) + switch { + case err == nil: + if hostCalendarEvent.WebhookStatus != fleet.CalendarWebhookStatusPending { + return nil + } + case fleet.IsNotFound(err): + return nil + default: + return ctxerr.Wrap(ctx, err, "get host calendar event") + } + + now := time.Now() + if now.Before(calendarEvent.StartTime) { + level.Warn(logger).Log("msg", "results came too early", "now", now, "start_time", calendarEvent.StartTime) + return nil + } + + // + // TODO(lucas): Discuss. + // + const allowedTimeBeforeEndTime = 5 * time.Minute // up to 5 minutes before the end_time + + if now.After(calendarEvent.EndTime.Add(-allowedTimeBeforeEndTime)) { + level.Warn(logger).Log("msg", "results came too late", "now", now, "end_time", calendarEvent.EndTime) + return nil + } + + calendarPolicies, err := ds.GetCalendarPolicies(ctx, *host.TeamID) + if err != nil { + return ctxerr.Wrap(ctx, err, "get calendar policy ids") + } + if len(calendarPolicies) == 0 { + return nil + } + + failingCalendarPolicies := getFailingCalendarPolicies(policyResults, calendarPolicies) + if len(failingCalendarPolicies) == 0 { + return nil + } + + go func() { + if err := fleet.FireCalendarWebhook( + team.Config.Integrations.GoogleCalendar.WebhookURL, + host.ID, host.HardwareSerial, host.DisplayName(), failingCalendarPolicies, "", + ); err != nil { + level.Error(logger).Log("msg", "fire webhook", "err", err) + return + } + if err := ds.UpdateHostCalendarWebhookStatus(context.Background(), host.ID, fleet.CalendarWebhookStatusSent); err != nil { + level.Error(logger).Log("msg", "mark fired webhook as sent", "err", err) + } + }() + + return nil +} + +func getFailingCalendarPolicies(policyResults map[uint]*bool, calendarPolicies []fleet.PolicyCalendarData) []fleet.PolicyCalendarData { + var failingPolicies []fleet.PolicyCalendarData + for _, calendarPolicy := range calendarPolicies { + result, ok := policyResults[calendarPolicy.ID] + if !ok || // ignore result of a policy that's not configured for calendar. + result == nil { // ignore policies that failed to execute. + continue + } + if !*result { + failingPolicies = append(failingPolicies, calendarPolicy) + } + } + return failingPolicies +} + // preProcessSoftwareResults will run pre-processing on the responses of the software queries. // It will move the results from the software extra queries (e.g. software_vscode_extensions) // into the main software query results (software_{macos|linux|windows}). From 196d8ce5b75c69f5ab12a60cfc8d5e2f0e8e3073 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 19 Mar 2024 16:19:38 -0500 Subject: [PATCH 10/36] Calendar interface updates and mock calendar (#17701) - Updated calendar interface to use updated `genBodyFn` - The mock calendar is enabled by specifying `calendar-mock@example.com` as the service account email. --- cmd/fleet/calendar_cron.go | 14 ++-- ee/server/calendar/google_calendar.go | 46 +++++++----- ee/server/calendar/google_calendar_mock.go | 81 ++++++++++++++++++++++ ee/server/calendar/google_calendar_test.go | 59 ++++++++++++---- server/fleet/calendar.go | 4 +- 5 files changed, 166 insertions(+), 38 deletions(-) create mode 100644 ee/server/calendar/google_calendar_mock.go diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index 099a938b2..e8ec7685d 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -231,9 +231,10 @@ func processFailingHostExistingCalendarEvent( calendarEvent *fleet.CalendarEvent, host fleet.HostPolicyMembershipData, ) error { - updatedEvent, updated, err := calendar.GetAndUpdateEvent(calendarEvent, func() string { - return generateCalendarEventBody(orgName, host.HostDisplayName) - }) + updatedEvent, updated, err := calendar.GetAndUpdateEvent( + calendarEvent, func(bool) string { + return generateCalendarEventBody(orgName, host.HostDisplayName) + }) if err != nil { return fmt.Errorf("get event calendar on db: %w", err) } @@ -325,9 +326,12 @@ func attemptCreatingEventOnUserCalendar( // - If it’s the 3rd Tuesday, Weds, Thurs, etc. of the month and it’s past the last slot, schedule the call for the next business day. year, month, today := time.Now().Date() preferredDate := getPreferredCalendarEventDate(year, month, today) - body := generateCalendarEventBody(orgName, host.HostDisplayName) for { - calendarEvent, err := userCalendar.CreateEvent(preferredDate, body) + calendarEvent, err := userCalendar.CreateEvent( + preferredDate, func(bool) string { + return generateCalendarEventBody(orgName, host.HostDisplayName) + }, + ) var dee fleet.DayEndedError switch { case err == nil: diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 53e189e24..91e1661a8 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -50,8 +50,12 @@ type GoogleCalendar struct { func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { if config.API == nil { - var lowLevelAPI GoogleCalendarAPI = &GoogleCalendarLowLevelAPI{} - config.API = lowLevelAPI + if config.IntegrationConfig.Email == "calendar-mock@example.com" { + // Assumes that only 1 Fleet server accesses the calendar, since all mock events are held in memory + config.API = &GoogleCalendarMockAPI{} + } else { + config.API = &GoogleCalendarLowLevelAPI{} + } } return &GoogleCalendar{ config: config, @@ -136,7 +140,9 @@ func (c *GoogleCalendar) Configure(userEmail string) error { return nil } -func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func() string) (*fleet.CalendarEvent, bool, error) { +func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn func(conflict bool) string) ( + *fleet.CalendarEvent, bool, error, +) { // We assume that the Fleet event has not already ended. We will simply return it if it has not been modified. details, err := c.unmarshalDetails(event) if err != nil { @@ -167,6 +173,10 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn if gEvent.End.DateTime == "" { // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. // We won't handle all-day events at this time, and treat the event as deleted. + err = c.DeleteEvent(event) + if err != nil { + level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err) + } deleted = true } @@ -194,6 +204,10 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn if gEvent.Start.DateTime == "" { // User has modified the event to be an all-day event. All-day events are problematic because they depend on the user's timezone. // We won't handle all-day events at this time, and treat the event as deleted. + err = c.DeleteEvent(event) + if err != nil { + level.Warn(c.config.Logger).Log("msg", "deleting Google calendar event which was changed to all-day event", "err", err) + } deleted = true } } @@ -212,7 +226,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn newStartDate := calculateNewEventDate(event.StartTime) - fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn()) + fleetEvent, err := c.CreateEvent(newStartDate, genBodyFn) if err != nil { return nil, false, err } @@ -269,13 +283,15 @@ func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDet return &details, nil } -func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, body string) (*fleet.CalendarEvent, error) { - return c.createEvent(dayOfEvent, body, time.Now) +func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, genBodyFn func(conflict bool) string) (*fleet.CalendarEvent, error) { + return c.createEvent(dayOfEvent, genBodyFn, time.Now) } // createEvent creates a new event on the calendar on the given date. timeNow is a function that returns the current time. // timeNow can be overwritten for testing -func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow func() time.Time) (*fleet.CalendarEvent, error) { +func (c *GoogleCalendar) createEvent( + dayOfEvent time.Time, genBodyFn func(conflict bool) string, timeNow func() time.Time, +) (*fleet.CalendarEvent, error) { if c.timezoneOffset == nil { err := getTimezone(c) if err != nil { @@ -310,6 +326,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow if err != nil { return nil, ctxerr.Wrap(c.config.Context, err, "listing Google calendar events") } + var conflict bool for _, gEvent := range events.Items { // Ignore cancelled events if gEvent.Status == "cancelled" { @@ -353,7 +370,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow if startTime.Before(eventEnd) { // Event occurs during our event, so we need to adjust. var isLastSlot bool - eventStart, eventEnd, isLastSlot = adjustEventTimes(*endTime, dayEnd) + eventStart, eventEnd, isLastSlot, conflict = adjustEventTimes(*endTime, dayEnd) if isLastSlot { break } @@ -367,7 +384,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow event.Start = &calendar.EventDateTime{DateTime: eventStart.Format(time.RFC3339)} event.End = &calendar.EventDateTime{DateTime: eventEnd.Format(time.RFC3339)} event.Summary = eventTitle - event.Description = body + event.Description = genBodyFn(conflict) event, err = c.config.API.CreateEvent(event) if err != nil { return nil, ctxerr.Wrap(c.config.Context, err, "creating Google calendar event") @@ -383,7 +400,7 @@ func (c *GoogleCalendar) createEvent(dayOfEvent time.Time, body string, timeNow return fleetEvent, nil } -func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time, eventEnd time.Time, isLastSlot bool) { +func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time, eventEnd time.Time, isLastSlot bool, conflict bool) { eventStart = endTime.Truncate(eventLength) if eventStart.Before(endTime) { eventStart = eventStart.Add(eventLength) @@ -394,11 +411,11 @@ func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time eventEnd = dayEnd eventStart = eventEnd.Add(-eventLength) isLastSlot = true - } - if eventEnd.Equal(dayEnd) { + conflict = true + } else if eventEnd.Equal(dayEnd) { isLastSlot = true } - return eventStart, eventEnd, isLastSlot + return eventStart, eventEnd, isLastSlot, conflict } func getTimezone(gCal *GoogleCalendar) error { @@ -444,9 +461,6 @@ func (c *GoogleCalendar) googleEventToFleetEvent(startTime time.Time, endTime ti } func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error { - if c.config == nil { - return errors.New("the Google calendar is not connected. Please call Configure first") - } details, err := c.unmarshalDetails(event) if err != nil { return err diff --git a/ee/server/calendar/google_calendar_mock.go b/ee/server/calendar/google_calendar_mock.go new file mode 100644 index 000000000..a6d7d6040 --- /dev/null +++ b/ee/server/calendar/google_calendar_mock.go @@ -0,0 +1,81 @@ +package calendar + +import ( + "context" + "errors" + kitlog "github.com/go-kit/log" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" + "net/http" + "os" + "strconv" + "sync" + "time" +) + +type GoogleCalendarMockAPI struct { + logger kitlog.Logger +} + +var events = make(map[string]*calendar.Event) +var mu sync.Mutex +var id uint64 + +const latency = 500 * time.Millisecond + +// Configure creates a new Google Calendar service using the provided credentials. +func (lowLevelAPI *GoogleCalendarMockAPI) Configure(_ context.Context, _ string, _ string, userToImpersonate string) error { + if lowLevelAPI.logger == nil { + lowLevelAPI.logger = kitlog.With(kitlog.NewLogfmtLogger(os.Stderr), "mock", "GoogleCalendarMockAPI", "user", userToImpersonate) + } + return nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) GetSetting(name string) (*calendar.Setting, error) { + time.Sleep(latency) + lowLevelAPI.logger.Log("msg", "GetSetting", "name", name) + if name == "timezone" { + return &calendar.Setting{ + Id: "timezone", + Value: "America/Chicago", + }, nil + } + return nil, errors.New("setting not supported") +} + +func (lowLevelAPI *GoogleCalendarMockAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + mu.Lock() + defer mu.Unlock() + id += 1 + event.Id = strconv.FormatUint(id, 10) + lowLevelAPI.logger.Log("msg", "CreateEvent", "id", event.Id, "start", event.Start.DateTime) + events[event.Id] = event + return event, nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) GetEvent(id, _ string) (*calendar.Event, error) { + time.Sleep(latency) + mu.Lock() + defer mu.Unlock() + event, ok := events[id] + if !ok { + return nil, &googleapi.Error{Code: http.StatusNotFound} + } + lowLevelAPI.logger.Log("msg", "GetEvent", "id", id, "start", event.Start.DateTime) + return event, nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) ListEvents(string, string) (*calendar.Events, error) { + time.Sleep(latency) + lowLevelAPI.logger.Log("msg", "ListEvents") + return &calendar.Events{}, nil +} + +func (lowLevelAPI *GoogleCalendarMockAPI) DeleteEvent(id string) error { + time.Sleep(latency) + mu.Lock() + defer mu.Unlock() + lowLevelAPI.logger.Log("msg", "DeleteEvent", "id", id) + delete(events, id) + return nil +} diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go index 4c3e2db09..cd3624275 100644 --- a/ee/server/calendar/google_calendar_test.go +++ b/ee/server/calendar/google_calendar_test.go @@ -162,7 +162,7 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { Etag: baseETag, // ETag matches -- no modifications to event }, nil } - genBodyFn := func() string { + genBodyFn := func(bool) string { t.Error("genBodyFn should not be called") return "event-body" } @@ -300,13 +300,14 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { return &calendar.Events{}, nil } - genBodyFn = func() string { + genBodyFn = func(conflict bool) string { + assert.False(t, conflict) return "event-body" } eventCreated := false mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { assert.Equal(t, eventTitle, event.Summary) - assert.Equal(t, genBodyFn(), event.Description) + assert.Equal(t, genBodyFn(false), event.Description) event.Id = baseEventID event.Etag = baseETag eventCreated = true @@ -345,6 +346,10 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { assert.True(t, eventCreated) // all day event (deleted) + mockAPI.DeleteEventFunc = func(id string) error { + assert.Equal(t, baseEventID, id) + return nil + } mockAPI.GetEventFunc = func(id, eTag string) (*calendar.Event, error) { return &calendar.Event{ Id: baseEventID, @@ -373,10 +378,6 @@ func TestGoogleCalendar_GetAndUpdateEvent(t *testing.T) { }, nil } eventCreated = false - mockAPI.DeleteEventFunc = func(id string) error { - assert.Equal(t, baseEventID, id) - return nil - } retrievedEvent, updated, err = cal.GetAndUpdateEvent(event, genBodyFn) require.NoError(t, err) assert.True(t, updated) @@ -411,13 +412,21 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { event.Etag = baseETag return event, nil } + genBodyFn := func(conflict bool) string { + assert.False(t, conflict) + return eventBody + } + genBodyConflictFn := func(conflict bool) string { + assert.True(t, conflict) + return eventBody + } // Happy path test -- empty calendar date := time.Now().Add(48 * time.Hour) location, _ := time.LoadLocation(tzId) expectedStartTime := time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) _, expectedOffset := expectedStartTime.Zone() - event, err := cal.CreateEvent(date, eventBody) + event, err := cal.CreateEvent(date, genBodyFn) require.NoError(t, err) assert.Equal(t, baseUserEmail, event.Email) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) @@ -434,7 +443,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { // Workday already ended date = time.Now().Add(-48 * time.Hour) - _, err = cal.CreateEvent(date, eventBody) + _, err = cal.CreateEvent(date, genBodyFn) assert.ErrorAs(t, err, &fleet.DayEndedError{}) // There is no time left in the day to schedule an event @@ -443,7 +452,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { now := time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 45, 0, 0, location) return now } - _, err = gCal.createEvent(date, eventBody, timeNow) + _, err = gCal.createEvent(date, genBodyFn, timeNow) assert.ErrorAs(t, err, &fleet.DayEndedError{}) // Workday already started @@ -452,7 +461,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { timeNow = func() time.Time { return expectedStartTime } - event, err = gCal.createEvent(date, eventBody, timeNow) + event, err = gCal.createEvent(date, genBodyFn, timeNow) require.NoError(t, err) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) @@ -545,7 +554,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { return gEvents, nil } expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), 12, 0, 0, 0, location) - event, err = gCal.CreateEvent(date, eventBody) + event, err = gCal.CreateEvent(date, genBodyFn) require.NoError(t, err) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) @@ -565,7 +574,27 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { return gEvents, nil } expectedStartTime = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) - event, err = gCal.CreateEvent(date, eventBody) + event, err = gCal.CreateEvent(date, genBodyConflictFn) + require.NoError(t, err) + assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) + assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) + + // Almost full schedule -- pick the last slot + date = time.Now().Add(48 * time.Hour) + dayStart = time.Date(date.Year(), date.Month(), date.Day(), startHour, 0, 0, 0, location) + dayEnd = time.Date(date.Year(), date.Month(), date.Day(), endHour-1, 30, 0, 0, location) + gEvents = &calendar.Events{} + gEvent = &calendar.Event{ + Id: "9-to-4-30-event-id", + Start: &calendar.EventDateTime{DateTime: dayStart.Format(time.RFC3339)}, + End: &calendar.EventDateTime{DateTime: dayEnd.Format(time.RFC3339)}, + } + gEvents.Items = append(gEvents.Items, gEvent) + mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { + return gEvents, nil + } + expectedStartTime = dayEnd + event, err = gCal.CreateEvent(date, genBodyFn) require.NoError(t, err) assert.Equal(t, expectedStartTime.UTC(), event.StartTime.UTC()) assert.Equal(t, expectedStartTime.Add(eventLength).UTC(), event.EndTime.UTC()) @@ -574,7 +603,7 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { mockAPI.ListEventsFunc = func(timeMin, timeMax string) (*calendar.Events, error) { return nil, assert.AnError } - _, err = gCal.CreateEvent(date, eventBody) + _, err = gCal.CreateEvent(date, genBodyFn) assert.ErrorIs(t, err, assert.AnError) // API error in CreateEvent @@ -584,6 +613,6 @@ func TestGoogleCalendar_CreateEvent(t *testing.T) { mockAPI.CreateEventFunc = func(event *calendar.Event) (*calendar.Event, error) { return nil, assert.AnError } - _, err = gCal.CreateEvent(date, eventBody) + _, err = gCal.CreateEvent(date, genBodyFn) assert.ErrorIs(t, err, assert.AnError) } diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index 9b45c2c8a..592fff430 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -21,11 +21,11 @@ type UserCalendar interface { // CreateEvent, GetAndUpdateEvent and DeleteEvent reference the user's calendar. Configure(userEmail string) error // CreateEvent creates a new event on the calendar on the given date. DayEndedError is returned if there is no time left on the given date to schedule event. - CreateEvent(dateOfEvent time.Time, body string) (event *CalendarEvent, err error) + CreateEvent(dateOfEvent time.Time, genBodyFn func(conflict bool) string) (event *CalendarEvent, err error) // GetAndUpdateEvent retrieves the event from the calendar. // If the event has been modified, it returns the updated event. // If the event has been deleted, it schedules a new event with given body callback and returns the new event. - GetAndUpdateEvent(event *CalendarEvent, genBodyFn func() string) (updatedEvent *CalendarEvent, updated bool, err error) + GetAndUpdateEvent(event *CalendarEvent, genBodyFn func(conflict bool) string) (updatedEvent *CalendarEvent, updated bool, err error) // DeleteEvent deletes the event with the given ID. DeleteEvent(event *CalendarEvent) error } From e4ba41ac85f8e2160e567f47cc141bc32a3f7784 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 19 Mar 2024 19:46:55 -0500 Subject: [PATCH 11/36] Latest changes to configs (#17724) - Remove email from team configs - Accept api_key_json for global config --- cmd/fleetctl/apply_test.go | 29 +------ cmd/fleetctl/gitops_test.go | 9 +- .../gitops/global_config_no_paths.yml | 8 +- .../testdata/gitops/team_config_no_paths.yml | 1 - ee/server/calendar/google_calendar.go | 3 +- ee/server/calendar/google_calendar_test.go | 6 +- ee/server/service/teams.go | 21 ++--- server/fleet/app.go | 10 +-- server/fleet/integrations.go | 45 +++++++--- server/service/appconfig_test.go | 5 +- server/service/integration_core_test.go | 87 +++++++++++-------- server/service/integration_enterprise_test.go | 10 +-- .../generated_files/appconfig.txt | 3 +- 13 files changed, 120 insertions(+), 117 deletions(-) diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index 655e2e57b..e30bd3939 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -143,17 +143,12 @@ func TestApplyTeamSpecs(t *testing.T) { } agentOpts := json.RawMessage(`{"config":{"foo":"bar"},"overrides":{"platforms":{"darwin":{"foo":"override"}}}}`) - googleCalEmail := "service-valid@example.com" ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { return &fleet.AppConfig{ AgentOptions: &agentOpts, MDM: fleet.MDM{EnabledAndConfigured: true}, Integrations: fleet.Integrations{ - GoogleCalendar: []*fleet.GoogleCalendarIntegration{ - { - Email: googleCalEmail, - }, - }, + GoogleCalendar: []*fleet.GoogleCalendarIntegration{{}}, }, }, nil } @@ -461,7 +456,6 @@ spec: name: team1 integrations: google_calendar: - email: `+googleCalEmail+` enable_calendar_events: true webhook_url: https://example.com/webhook `, @@ -470,31 +464,11 @@ spec: require.NotNil(t, teamsByName["team1"].Config.Integrations.GoogleCalendar) assert.Equal( t, fleet.TeamGoogleCalendarIntegration{ - Email: googleCalEmail, Enable: true, WebhookURL: "https://example.com/webhook", }, *teamsByName["team1"].Config.Integrations.GoogleCalendar, ) - // Apply calendar integration -- invalid email - filename = writeTmpYml( - t, ` -apiVersion: v1 -kind: team -spec: - team: - name: team1 - integrations: - google_calendar: - email: not_present_globally@example.com - enable_calendar_events: true - webhook_url: https://example.com/webhook -`, - ) - - _, err = runAppNoChecks([]string{"apply", "-f", filename}) - assert.ErrorContains(t, err, "email must match a global Google Calendar integration email") - // Apply calendar integration -- invalid webhook destination filename = writeTmpYml( t, ` @@ -505,7 +479,6 @@ spec: name: team1 integrations: google_calendar: - email: `+googleCalEmail+` enable_calendar_events: true webhook_url: bozo `, diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go index c6013aa95..6a8fab648 100644 --- a/cmd/fleetctl/gitops_test.go +++ b/cmd/fleetctl/gitops_test.go @@ -361,7 +361,7 @@ func TestFullGlobalGitOps(t *testing.T) { assert.Len(t, appliedMacProfiles, 1) assert.Len(t, appliedWinProfiles, 1) require.Len(t, savedAppConfig.Integrations.GoogleCalendar, 1) - assert.Equal(t, "service@example.com", savedAppConfig.Integrations.GoogleCalendar[0].Email) + assert.Equal(t, "service@example.com", savedAppConfig.Integrations.GoogleCalendar[0].ApiKey["client_email"]) } func TestFullTeamGitOps(t *testing.T) { @@ -392,11 +392,7 @@ func TestFullTeamGitOps(t *testing.T) { WindowsEnabledAndConfigured: true, }, Integrations: fleet.Integrations{ - GoogleCalendar: []*fleet.GoogleCalendarIntegration{ - { - Email: "service@example.com", - }, - }, + GoogleCalendar: []*fleet.GoogleCalendarIntegration{{}}, }, }, nil } @@ -546,7 +542,6 @@ func TestFullTeamGitOps(t *testing.T) { assert.True(t, savedTeam.Config.WebhookSettings.HostStatusWebhook.Enable) assert.Equal(t, "https://example.com/host_status_webhook", savedTeam.Config.WebhookSettings.HostStatusWebhook.DestinationURL) require.NotNil(t, savedTeam.Config.Integrations.GoogleCalendar) - assert.Equal(t, "service@example.com", savedTeam.Config.Integrations.GoogleCalendar.Email) assert.True(t, savedTeam.Config.Integrations.GoogleCalendar.Enable) // Now clear the settings diff --git a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml index cf2eeece6..b487bf46e 100644 --- a/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/global_config_no_paths.yml @@ -138,9 +138,11 @@ org_settings: jira: [] zendesk: [] google_calendar: - - email: service@example.com - private_key: google_calendar_private_key - domain: example.com + - domain: example.com + api_key_json: { + "client_email": "service@example.com", + "private_key": "google_calendar_private_key", + } mdm: apple_bm_default_team: "" end_user_authentication: diff --git a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml index 58564a7bc..4785c72a7 100644 --- a/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml +++ b/cmd/fleetctl/testdata/gitops/team_config_no_paths.yml @@ -17,7 +17,6 @@ team_settings: host_expiry_window: 30 integrations: google_calendar: - email: service@example.com enable_calendar_events: true webhook_url: https://example.com/google_calendar_webhook agent_options: diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 91e1661a8..9c0c14707 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -131,7 +131,8 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { func (c *GoogleCalendar) Configure(userEmail string) error { err := c.config.API.Configure( - c.config.Context, c.config.IntegrationConfig.Email, c.config.IntegrationConfig.PrivateKey, userEmail, + c.config.Context, c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail], + c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarPrivateKey], userEmail, ) if err != nil { return ctxerr.Wrap(c.config.Context, err, "creating Google calendar service") diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go index cd3624275..ad5e1c89c 100644 --- a/ee/server/calendar/google_calendar_test.go +++ b/ee/server/calendar/google_calendar_test.go @@ -93,8 +93,10 @@ func makeConfig(mockAPI *MockGoogleCalendarLowLevelAPI) *GoogleCalendarConfig { config := &GoogleCalendarConfig{ Context: context.Background(), IntegrationConfig: &fleet.GoogleCalendarIntegration{ - Email: baseServiceEmail, - PrivateKey: basePrivateKey, + ApiKey: map[string]string{ + fleet.GoogleCalendarEmail: baseServiceEmail, + fleet.GoogleCalendarPrivateKey: basePrivateKey, + }, }, Logger: logger, API: mockAPI, diff --git a/ee/server/service/teams.go b/ee/server/service/teams.go index e0f503b38..fe81963ab 100644 --- a/ee/server/service/teams.go +++ b/ee/server/service/teams.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "github.com/fleetdm/fleet/v4/server" "github.com/fleetdm/fleet/v4/server/authz" @@ -214,7 +213,7 @@ func (svc *Service) ModifyTeam(ctx context.Context, teamID uint, payload fleet.T // Only update the calendar integration if it's not nil if payload.Integrations.GoogleCalendar != nil { invalid := &fleet.InvalidArgumentError{} - _ = svc.validateTeamCalendarIntegrations(ctx, team, payload.Integrations.GoogleCalendar, appCfg, invalid) + _ = svc.validateTeamCalendarIntegrations(payload.Integrations.GoogleCalendar, appCfg, invalid) if invalid.HasErrors() { return nil, ctxerr.Wrap(ctx, invalid) } @@ -1083,7 +1082,7 @@ func (svc *Service) editTeamFromSpec( } if spec.Integrations.GoogleCalendar != nil { - err = svc.validateTeamCalendarIntegrations(ctx, team, spec.Integrations.GoogleCalendar, appCfg, invalid) + err = svc.validateTeamCalendarIntegrations(spec.Integrations.GoogleCalendar, appCfg, invalid) if err != nil { return ctxerr.Wrap(ctx, err, "validate team calendar integrations") } @@ -1157,23 +1156,15 @@ func (svc *Service) editTeamFromSpec( } func (svc *Service) validateTeamCalendarIntegrations( - ctx context.Context, team *fleet.Team, calendarIntegration *fleet.TeamGoogleCalendarIntegration, + calendarIntegration *fleet.TeamGoogleCalendarIntegration, appCfg *fleet.AppConfig, invalid *fleet.InvalidArgumentError, ) error { if !calendarIntegration.Enable { return nil } - // Validate email - emailValid := false - calendarIntegration.Email = strings.TrimSpace(calendarIntegration.Email) - for _, globalCals := range appCfg.Integrations.GoogleCalendar { - if globalCals.Email == calendarIntegration.Email { - emailValid = true - break - } - } - if !emailValid { - invalid.Append("integrations.google_calendar.email", "email must match a global Google Calendar integration email") + // Check that global configs exist + if len(appCfg.Integrations.GoogleCalendar) == 0 { + invalid.Append("integrations.google_calendar.enable_calendar_events", "global Google Calendar integration is not configured") } // Validate URL if u, err := url.ParseRequestURI(calendarIntegration.WebhookURL); err != nil { diff --git a/server/fleet/app.go b/server/fleet/app.go index 560b8bb34..e1720e596 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "maps" "net/url" "reflect" "regexp" @@ -488,9 +489,6 @@ func (c *AppConfig) Obfuscate() { for _, zdIntegration := range c.Integrations.Zendesk { zdIntegration.APIToken = MaskedPassword } - for _, calIntegration := range c.Integrations.GoogleCalendar { - calIntegration.PrivateKey = MaskedPassword - } } // Clone implements cloner. @@ -574,8 +572,10 @@ func (c *AppConfig) Copy() *AppConfig { if len(c.Integrations.GoogleCalendar) > 0 { clone.Integrations.GoogleCalendar = make([]*GoogleCalendarIntegration, len(c.Integrations.GoogleCalendar)) for i, g := range c.Integrations.GoogleCalendar { - gc := *g - clone.Integrations.GoogleCalendar[i] = &gc + gCal := *g + clone.Integrations.GoogleCalendar[i] = &gCal + clone.Integrations.GoogleCalendar[i].ApiKey = make(map[string]string, len(g.ApiKey)) + maps.Copy(clone.Integrations.GoogleCalendar[i].ApiKey, g.ApiKey) } } diff --git a/server/fleet/integrations.go b/server/fleet/integrations.go index 8c3450948..18cce7182 100644 --- a/server/fleet/integrations.go +++ b/server/fleet/integrations.go @@ -112,7 +112,6 @@ func (z TeamZendeskIntegration) UniqueKey() string { } type TeamGoogleCalendarIntegration struct { - Email string `json:"email"` Enable bool `json:"enable_calendar_events"` WebhookURL string `json:"webhook_url"` } @@ -342,10 +341,14 @@ func makeTestZendeskRequest(ctx context.Context, intg *ZendeskIntegration) error return nil } +const ( + GoogleCalendarEmail = "client_email" + GoogleCalendarPrivateKey = "private_key" +) + type GoogleCalendarIntegration struct { - Email string `json:"email"` - PrivateKey string `json:"private_key"` - Domain string `json:"domain"` + Domain string `json:"domain"` + ApiKey map[string]string `json:"api_key_json"` } // Integrations configures the integrations with external systems. @@ -378,13 +381,35 @@ func ValidateGoogleCalendarIntegrations(intgs []*GoogleCalendarIntegration, inva invalid.Append("integrations.google_calendar", "integrating with >1 Google Workspace service account is not yet supported.") } for _, intg := range intgs { - intg.Email = strings.TrimSpace(intg.Email) - if intg.Email == "" { - invalid.Append("integrations.google_calendar.email", "email is required") + if email, ok := intg.ApiKey[GoogleCalendarEmail]; !ok { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarEmail), + fmt.Sprintf("%s is required", GoogleCalendarEmail), + ) + } else { + email = strings.TrimSpace(email) + intg.ApiKey[GoogleCalendarEmail] = email + if email == "" { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarEmail), + fmt.Sprintf("%s cannot be blank", GoogleCalendarEmail), + ) + } } - intg.PrivateKey = strings.TrimSpace(intg.PrivateKey) - if intg.PrivateKey == "" || intg.PrivateKey == MaskedPassword { - invalid.Append("integrations.google_calendar.private_key", "private_key is required") + if privateKey, ok := intg.ApiKey["private_key"]; !ok { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarPrivateKey), + fmt.Sprintf("%s is required", GoogleCalendarPrivateKey), + ) + } else { + privateKey = strings.TrimSpace(privateKey) + intg.ApiKey[GoogleCalendarPrivateKey] = privateKey + if privateKey == "" { + invalid.Append( + fmt.Sprintf("integrations.google_calendar.api_key_json.%s", GoogleCalendarPrivateKey), + fmt.Sprintf("%s cannot be blank", GoogleCalendarPrivateKey), + ) + } } intg.Domain = strings.TrimSpace(intg.Domain) if intg.Domain == "" { diff --git a/server/service/appconfig_test.go b/server/service/appconfig_test.go index e23f3b18b..8881b1965 100644 --- a/server/service/appconfig_test.go +++ b/server/service/appconfig_test.go @@ -490,7 +490,7 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { {APIToken: "zendesktoken"}, }, GoogleCalendar: []*fleet.GoogleCalendarIntegration{ - {PrivateKey: "google-calendar-private-key"}, + {ApiKey: map[string]string{fleet.GoogleCalendarPrivateKey: "google-calendar-private-key"}}, }, }, }, nil @@ -569,7 +569,8 @@ func TestAppConfigSecretsObfuscated(t *testing.T) { require.Equal(t, ac.SMTPSettings.SMTPPassword, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Jira[0].APIToken, fleet.MaskedPassword) require.Equal(t, ac.Integrations.Zendesk[0].APIToken, fleet.MaskedPassword) - require.Equal(t, ac.Integrations.GoogleCalendar[0].PrivateKey, fleet.MaskedPassword) + // Google Calendar private key is not obfuscated + require.Equal(t, ac.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarPrivateKey], "google-calendar-private-key") } }) } diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index a02913a4e..b531fd90b 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -5181,8 +5181,10 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": %q }] } @@ -5192,8 +5194,8 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { appConfig := s.getConfig() require.Len(t, appConfig.Integrations.GoogleCalendar, 1) - assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) - assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarEmail]) + assert.Equal(t, privateKey, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarPrivateKey]) assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) // Add 2nd config -- not allowed at this time @@ -5202,18 +5204,22 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": %q }, { - "email": "bozo@example.com"", - "private_key": "abc", + "api_key_json": { + "client_email": "bozo@example.com", + "private_key": "abc" + }, "domain": "example.com" }] } }`, email, privateKey, domain, - )), http.StatusBadRequest, + )), http.StatusUnprocessableEntity, ) // Make an unrelated config change, should not remove the integrations @@ -5237,8 +5243,10 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": %q }] } @@ -5247,8 +5255,8 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { ) appConfig = s.getConfig() require.Len(t, appConfig.Integrations.GoogleCalendar, 1) - assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].Email) - assert.Equal(t, fleet.MaskedPassword, appConfig.Integrations.GoogleCalendar[0].PrivateKey) + assert.Equal(t, email, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarEmail]) + assert.Equal(t, privateKey, appConfig.Integrations.GoogleCalendar[0].ApiKey[fleet.GoogleCalendarPrivateKey]) assert.Equal(t, domain, appConfig.Integrations.GoogleCalendar[0].Domain) // Clearing other integrations does not clear Google Calendar integration @@ -5284,7 +5292,9 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, + "api_key_json": { + "client_email": %q + }, "domain": %q }] } @@ -5292,29 +5302,16 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { )), http.StatusUnprocessableEntity, ) - // Try adding Google Calendar integration with masked private key -- not allowed - s.DoRaw( - "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( - `{ - "integrations": { - "google_calendar": [{ - "email": %q, - "private_key": %q, - "domain": %q - }] - } - }`, email, fleet.MaskedPassword, domain, - )), http.StatusUnprocessableEntity, - ) - // Empty email -- not allowed s.DoRaw( "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( `{ "integrations": { "google_calendar": [{ - "email": " ", - "private_key": %q, + "api_key_json": { + "client_email": " ", + "private_key": %q + }, "domain": %q }] } @@ -5328,8 +5325,10 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, + "api_key_json": { + "client_email": %q, + "private_key": %q + }, "domain": "" }] } @@ -5343,9 +5342,11 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": %q, - "domain": %q + "api_key_json": { + "client_email": %q, + "private_key": %q + }, + "domain": %q, "foo": "bar" }] } @@ -5353,6 +5354,20 @@ func (s *integrationTestSuite) TestGoogleCalendarIntegrations() { )), http.StatusBadRequest, ) + // Null api_key_json -- fails validation + s.DoRaw( + "PATCH", "/api/v1/fleet/config", []byte(fmt.Sprintf( + `{ + "integrations": { + "google_calendar": [{ + "api_key_json": null, + "domain": %q + }] + } + }`, domain, + )), http.StatusUnprocessableEntity, + ) + } func (s *integrationTestSuite) TestQueriesBadRequests() { diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 22c2779ca..7466d78d1 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -105,8 +105,10 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { `{ "integrations": { "google_calendar": [{ - "email": %q, - "private_key": "testKey", + "api_key_json": { + "client_email": %q, + "private_key": "testKey" + }, "domain": "example.com" }] } @@ -197,7 +199,6 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { "name": teamName, "integrations": map[string]any{ "google_calendar": map[string]any{ - "email": calendarEmail, "enable_calendar_events": true, "webhook_url": calendarWebhookUrl, }, @@ -210,7 +211,6 @@ func (s *integrationEnterpriseTestSuite) TestTeamSpecs() { team, err = s.ds.TeamByName(context.Background(), teamName) require.NotNil(t, team.Config.Integrations.GoogleCalendar) - assert.Equal(t, calendarEmail, team.Config.Integrations.GoogleCalendar.Email) assert.Equal(t, calendarWebhookUrl, team.Config.Integrations.GoogleCalendar.WebhookURL) assert.True(t, team.Config.Integrations.GoogleCalendar.Enable) @@ -1026,7 +1026,7 @@ func (s *integrationEnterpriseTestSuite) TestTeamEndpoints() { modifyCalendar := fleet.TeamPayload{ Integrations: &fleet.TeamIntegrations{ GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ - Email: "calendar@example.com", + WebhookURL: "https://example.com/modified", }, }, } diff --git a/tools/cloner-check/generated_files/appconfig.txt b/tools/cloner-check/generated_files/appconfig.txt index 7deb8d10f..9a03bf15b 100644 --- a/tools/cloner-check/generated_files/appconfig.txt +++ b/tools/cloner-check/generated_files/appconfig.txt @@ -85,9 +85,8 @@ github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration GroupID int64 github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableFailingPolicies bool github.com/fleetdm/fleet/v4/server/fleet/ZendeskIntegration EnableSoftwareVulnerabilities bool github.com/fleetdm/fleet/v4/server/fleet/Integrations GoogleCalendar []*fleet.GoogleCalendarIntegration -github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Email string -github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration PrivateKey string github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration Domain string +github.com/fleetdm/fleet/v4/server/fleet/GoogleCalendarIntegration ApiKey map[string]string github.com/fleetdm/fleet/v4/server/fleet/AppConfig MDM fleet.MDM github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMDefaultTeam string github.com/fleetdm/fleet/v4/server/fleet/MDM AppleBMEnabledAndConfigured bool From d97e32fc219775c5450b6bc084d12098c4ce234f Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 19 Mar 2024 19:58:48 -0500 Subject: [PATCH 12/36] Fix compile issue due to merge. --- ee/server/calendar/google_calendar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 9c0c14707..26d1ba1e6 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -50,7 +50,7 @@ type GoogleCalendar struct { func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { if config.API == nil { - if config.IntegrationConfig.Email == "calendar-mock@example.com" { + if config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == "calendar-mock@example.com" { // Assumes that only 1 Fleet server accesses the calendar, since all mock events are held in memory config.API = &GoogleCalendarMockAPI{} } else { From 4db06f2cbb1734e3da79f9253babb97e9fc4e49a Mon Sep 17 00:00:00 2001 From: RachelElysia <71795832+RachelElysia@users.noreply.github.com> Date: Wed, 20 Mar 2024 16:07:27 -0400 Subject: [PATCH 13/36] Fleet Calendar feature: Updates to manage automations modal (#17652) --- .../ManagePoliciesPage/ManagePoliciesPage.tsx | 11 +- .../ExamplePayload/ExamplePayload.tsx | 64 +++++++ .../components/ExamplePayload/_styles.scss | 9 + .../components/ExamplePayload/index.ts | 1 + .../ExampleTicket.tsx} | 42 +---- .../components/ExampleTicket/_styles.scss | 10 ++ .../components/ExampleTicket/index.ts | 1 + .../ManagePolicyAutomationsModal/index.ts | 1 - .../OtherWorkflowsModal.tsx} | 161 +++++++++--------- .../_styles.scss | 9 +- .../components/OtherWorkflowsModal/index.ts | 1 + .../PreviewPayloadModal.tsx | 97 ----------- .../PreviewPayloadModal/_styles.scss | 54 ------ .../components/PreviewPayloadModal/index.ts | 1 - .../PreviewTicketModal/_styles.scss | 13 -- .../components/PreviewTicketModal/index.ts | 1 - 16 files changed, 185 insertions(+), 291 deletions(-) create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts rename frontend/pages/policies/ManagePoliciesPage/components/{PreviewTicketModal/PreviewTicketModal.tsx => ExampleTicket/ExampleTicket.tsx} (52%) create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts rename frontend/pages/policies/ManagePoliciesPage/components/{ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx => OtherWorkflowsModal/OtherWorkflowsModal.tsx} (82%) rename frontend/pages/policies/ManagePoliciesPage/components/{ManagePolicyAutomationsModal => OtherWorkflowsModal}/_styles.scss (69%) create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/index.ts delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/PreviewPayloadModal.tsx delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss delete mode 100644 frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx index be95e222d..ce359b577 100644 --- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx @@ -41,7 +41,7 @@ import TableDataError from "components/DataError"; import MainContent from "components/MainContent"; import PoliciesTable from "./components/PoliciesTable"; -import ManagePolicyAutomationsModal from "./components/ManagePolicyAutomationsModal"; +import OtherWorkflowsModal from "./components/OtherWorkflowsModal"; import AddPolicyModal from "./components/AddPolicyModal"; import DeletePolicyModal from "./components/DeletePolicyModal"; @@ -129,7 +129,6 @@ const ManagePolicyPage = ({ const [showManageAutomationsModal, setShowManageAutomationsModal] = useState( false ); - const [showPreviewPayloadModal, setShowPreviewPayloadModal] = useState(false); const [showAddPolicyModal, setShowAddPolicyModal] = useState(false); const [showDeletePolicyModal, setShowDeletePolicyModal] = useState(false); @@ -477,10 +476,6 @@ const ManagePolicyPage = ({ const toggleManageAutomationsModal = () => setShowManageAutomationsModal(!showManageAutomationsModal); - const togglePreviewPayloadModal = useCallback(() => { - setShowPreviewPayloadModal(!showPreviewPayloadModal); - }, [setShowPreviewPayloadModal, showPreviewPayloadModal]); - const toggleAddPolicyModal = () => setShowAddPolicyModal(!showAddPolicyModal); const toggleDeletePolicyModal = () => @@ -796,15 +791,13 @@ const ManagePolicyPage = ({ )} {config && automationsConfig && showManageAutomationsModal && ( - )} {showAddPolicyModal && ( diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx new file mode 100644 index 000000000..4dc1c2462 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/ExamplePayload.tsx @@ -0,0 +1,64 @@ +import React, { useContext } from "react"; +import { syntaxHighlight } from "utilities/helpers"; + +import { AppContext } from "context/app"; +import { IPolicyWebhookPreviewPayload } from "interfaces/policy"; + +const baseClass = "example-payload"; + +interface IHostPreview { + id: number; + display_name: string; + url: string; +} + +interface IExamplePayload { + timestamp: string; + policy: IPolicyWebhookPreviewPayload; + hosts: IHostPreview[]; +} + +const ExamplePayload = (): JSX.Element => { + const { isFreeTier } = useContext(AppContext); + + const json: IExamplePayload = { + timestamp: "0000-00-00T00:00:00Z", + policy: { + id: 1, + name: "Is Gatekeeper enabled?", + query: "SELECT 1 FROM gatekeeper WHERE assessments_enabled = 1;", + description: "Checks if gatekeeper is enabled on macOS devices.", + author_id: 1, + author_name: "John", + author_email: "john@example.com", + resolution: "Turn on Gatekeeper feature in System Preferences.", + passing_host_count: 2000, + failing_host_count: 300, + critical: false, + }, + hosts: [ + { + id: 1, + display_name: "macbook-1", + url: "https://fleet.example.com/hosts/1", + }, + { + id: 2, + display_name: "macbbook-2", + url: "https://fleet.example.com/hosts/2", + }, + ], + }; + if (isFreeTier) { + delete json.policy.critical; + } + + return ( +
+ 
POST https://server.com/example
+ 
+
+ ); +}; + +export default ExamplePayload; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss new file mode 100644 index 000000000..2297445b9 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/_styles.scss @@ -0,0 +1,9 @@ +.example-payload { + display: flex; + flex-direction: column; + gap: $pad-large; + + pre { + margin: 0; + } +} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts new file mode 100644 index 000000000..a9ab7d050 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExamplePayload/index.ts @@ -0,0 +1 @@ +export { default } from "./ExamplePayload"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/PreviewTicketModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/ExampleTicket.tsx similarity index 52% rename from frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/PreviewTicketModal.tsx rename to frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/ExampleTicket.tsx index c46d2e4e5..1ff58279f 100644 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/PreviewTicketModal.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/ExampleTicket.tsx @@ -1,28 +1,24 @@ import React, { useContext } from "react"; import { AppContext } from "context/app"; -import Modal from "components/Modal"; -import Button from "components/buttons/Button"; -import CustomLink from "components/CustomLink"; import { IIntegrationType } from "interfaces/integration"; +import Card from "components/Card"; import JiraPreview from "../../../../../../assets/images/jira-policy-automation-preview-400x419@2x.png"; import ZendeskPreview from "../../../../../../assets/images/zendesk-policy-automation-preview-400x515@2x.png"; import JiraPreviewPremium from "../../../../../../assets/images/jira-policy-automation-preview-premium-400x316@2x.png"; import ZendeskPreviewPremium from "../../../../../../assets/images/zendesk-policy-automation-preview-premium-400x483@2x.png"; -const baseClass = "preview-ticket-modal"; +const baseClass = "example-ticket"; -interface IPreviewTicketModalProps { +interface IExampleTicketProps { integrationType?: IIntegrationType; - onCancel: () => void; } -const PreviewTicketModal = ({ +const ExampleTicket = ({ integrationType, - onCancel, -}: IPreviewTicketModalProps): JSX.Element => { +}: IExampleTicketProps): JSX.Element => { const { isPremiumTier } = useContext(AppContext); const screenshot = @@ -41,30 +37,10 @@ const PreviewTicketModal = ({ ); return ( - -
-

- Want to learn more about how automations in Fleet work?{" "} - -

-
{screenshot}
-
- -
-
- + + {screenshot} + ); }; -export default PreviewTicketModal; +export default ExampleTicket; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss new file mode 100644 index 000000000..4212f33fa --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/_styles.scss @@ -0,0 +1,10 @@ +.example-ticket { + display: flex; + flex-direction: column; + align-items: center; + box-sizing: border-box; + + &__screenshot { + max-width: 400px; + } +} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts new file mode 100644 index 000000000..355709708 --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/ExampleTicket/index.ts @@ -0,0 +1 @@ +export { default } from "./ExampleTicket"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts deleted file mode 100644 index d8e2cefbc..000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./ManagePolicyAutomationsModal"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx similarity index 82% rename from frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx rename to frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx index ae71c22cf..907f4edb4 100644 --- a/frontend/pages/policies/ManagePoliciesPage/components/ManagePolicyAutomationsModal/ManagePolicyAutomationsModal.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx @@ -19,22 +19,21 @@ import Dropdown from "components/forms/fields/Dropdown"; import InputField from "components/forms/fields/InputField"; import Radio from "components/forms/fields/Radio"; import validUrl from "components/forms/validators/valid_url"; +import RevealButton from "components/buttons/RevealButton"; +import CustomLink from "components/CustomLink"; +import ExampleTicket from "../ExampleTicket"; +import ExamplePayload from "../ExamplePayload"; -import PreviewPayloadModal from "../PreviewPayloadModal"; -import PreviewTicketModal from "../PreviewTicketModal"; - -interface IManagePolicyAutomationsModalProps { +interface IOtherWorkflowsModalProps { automationsConfig: IAutomationsConfig | ITeamAutomationsConfig; availableIntegrations: IIntegrations; availablePolicies: IPolicy[]; isUpdatingAutomations: boolean; - showPreviewPayloadModal: boolean; onExit: () => void; handleSubmit: (formData: { webhook_settings: Pick; integrations: IIntegrations; }) => void; - togglePreviewPayloadModal: () => void; } interface ICheckedPolicy { @@ -83,18 +82,16 @@ const useCheckboxListStateManagement = ( return { policyItems, updatePolicyItems }; }; -const baseClass = "manage-policy-automations-modal"; +const baseClass = "other-workflows-modal"; -const ManagePolicyAutomationsModal = ({ +const OtherWorkflowsModal = ({ automationsConfig, availableIntegrations, availablePolicies, isUpdatingAutomations, - showPreviewPayloadModal, onExit, handleSubmit, - togglePreviewPayloadModal: togglePreviewModal, -}: IManagePolicyAutomationsModalProps): JSX.Element => { +}: IOtherWorkflowsModalProps): JSX.Element => { const { webhook_settings: { failing_policies_webhook: webhook }, } = automationsConfig; @@ -131,6 +128,9 @@ const ManagePolicyAutomationsModal = ({ IIntegration | undefined >(serverEnabledIntegration); + const [showExamplePayload, setShowExamplePayload] = useState(false); + const [showExampleTicket, setShowExampleTicket] = useState(false); + const [errors, setErrors] = useState<{ [key: string]: string }>({}); const { policyItems, updatePolicyItems } = useCheckboxListStateManagement( @@ -218,13 +218,6 @@ const ManagePolicyAutomationsModal = ({ z.group_id === selectedIntegration?.group_id, })) || null; - // if ( - // !isPolicyAutomationsEnabled || - // (!isWebhookEnabled && !selectedIntegration) - // ) { - // newPolicyIds = []; - // } - const updatedEnabledPoliciesAcrossPages = () => { if (webhook.policy_ids) { // Array of policy ids on the page @@ -297,34 +290,52 @@ const ManagePolicyAutomationsModal = ({ placeholder="https://server.com/example" tooltip="Provide a URL to deliver a webhook request to." /> - + setShowExamplePayload(!showExamplePayload)} + /> + {showExamplePayload && } ); }; const renderIntegrations = () => { return jira?.length || zendesk?.length ? ( -
- +
+ +
+ setShowExampleTicket(!showExampleTicket)} /> - -
+ {showExampleTicket && ( + + )} + ) : (
You have no integrations.
@@ -338,22 +349,10 @@ const ManagePolicyAutomationsModal = ({ ); }; - const renderPreview = () => - !isWebhookEnabled ? ( - - ) : ( - - ); - - return showPreviewPayloadModal ? ( - renderPreview() - ) : ( + return ( @@ -372,12 +371,32 @@ const ManagePolicyAutomationsModal = ({ isPolicyAutomationsEnabled ? "enabled" : "disabled" }`} > +
+
Workflow
+ + +
+ {isWebhookEnabled ? renderWebhook() : renderIntegrations()}
{availablePolicies?.length ? ( <> -
- Choose which policies you would like to listen to: -
+
Policies:
{policyItems && policyItems.map((policyItem) => { const { isChecked, name, id } = policyItem; @@ -405,28 +424,14 @@ const ManagePolicyAutomationsModal = ({ )}
-
-
Workflow
- + The workflow will be triggered when hosts fail these policies.{" "} + - -
- {isWebhookEnabled ? renderWebhook() : renderIntegrations()} +

-
- - - ); -}; - -export default PreviewPayloadModal; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss deleted file mode 100644 index 0ff66f034..000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/_styles.scss +++ /dev/null @@ -1,54 +0,0 @@ -.preview-payload-modal { - &__sandbox-info { - margin-top: $pad-medium; - - p { - margin: 0; - margin-bottom: $pad-medium; - } - - p:last-child { - margin-bottom: 0; - } - } - - &__info-header { - font-weight: $bold; - } - - &__advanced-options-button { - margin: $pad-medium 0; - color: $core-vibrant-blue; - font-weight: $bold; - font-size: $x-small; - } - - .downcaret { - &::after { - content: url("../assets/images/icon-chevron-blue-16x16@2x.png"); - transform: scale(0.5); - width: 16px; - border-radius: 0px; - padding: 0px; - padding-left: 2px; - margin-bottom: 2px; - } - } - - .upcaret { - &::after { - content: url("../assets/images/icon-chevron-blue-16x16@2x.png"); - transform: scale(0.5) rotate(180deg); - width: 16px; - border-radius: 0px; - padding: 0px; - padding-left: 2px; - margin-bottom: 4px; - margin-left: 14px; - } - } - - .Select-value-label { - font-size: $small; - } -} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts deleted file mode 100644 index bc08b4723..000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewPayloadModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./PreviewPayloadModal"; diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss deleted file mode 100644 index 9024a7320..000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/_styles.scss +++ /dev/null @@ -1,13 +0,0 @@ -.preview-ticket-modal { - &__example { - display: flex; - justify-content: center; - } - - &__screenshot { - width: 400px; - height: auto; - border-radius: 8px; - filter: drop-shadow(0px 4px 16px rgba(0, 0, 0, 0.1)); - } -} diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts deleted file mode 100644 index 4d8716d44..000000000 --- a/frontend/pages/policies/ManagePoliciesPage/components/PreviewTicketModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from "./PreviewTicketModal"; From 5137fe380c07c3ed89c5e1b0f3b6f01b54b27490 Mon Sep 17 00:00:00 2001 From: Jacob Shandling <61553566+jacobshandling@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:53:34 -0700 Subject: [PATCH 14/36] 17445 calendar events modal (#17717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses #17445 Follow-up iteration: - Finalize styling of dropdown tooltips - All `//TODO`s Screenshot 2024-03-20 at 1 43 54 PM Screenshot 2024-03-20 at 1 44 01 PM Screenshot 2024-03-20 at 1 44 21 PM Screenshot 2024-03-20 at 1 44 26 PM Screenshot 2024-03-20 at 1 44 33 PM Screenshot 2024-03-20 at 1 44 43 PM Screenshot 2024-03-20 at 1 44 47 PM Screenshot 2024-03-20 at 1 45 02 PM Screenshot 2024-03-20 at 1 45 10 PM --------- Co-authored-by: Jacob Shandling --- frontend/__mocks__/configMock.ts | 1 + frontend/__mocks__/policyMock.ts | 1 + .../components/forms/FormField/FormField.tsx | 8 +- .../forms/fields/InputField/InputField.jsx | 3 + .../components/forms/fields/Slider/Slider.tsx | 5 +- .../graphics/CalendarEventPreview.tsx | 1184 +++++++++++++++++ frontend/components/graphics/index.ts | 2 + .../hooks/useCheckboxListStateManagement.tsx | 36 + frontend/interfaces/config.ts | 4 +- frontend/interfaces/integration.ts | 24 + frontend/interfaces/policy.ts | 2 + frontend/interfaces/team.ts | 4 +- .../HostActionsDropdown/_styles.scss | 105 +- .../HostDetailsPage/HostDetailsPage.tsx | 1 - .../ManagePoliciesPage/ManagePoliciesPage.tsx | 181 ++- .../policies/ManagePoliciesPage/_styles.scss | 28 +- .../CalendarEventsModal.tsx | 318 +++++ .../CalendarEventsModal/_styles.scss | 35 + .../components/CalendarEventsModal/index.ts | 1 + .../PoliciesTable/PoliciesTableConfig.tsx | 10 - frontend/services/entities/team_policies.ts | 2 + frontend/services/entities/teams.ts | 7 +- frontend/styles/var/mixins.scss | 100 ++ 23 files changed, 1917 insertions(+), 145 deletions(-) create mode 100644 frontend/components/graphics/CalendarEventPreview.tsx create mode 100644 frontend/hooks/useCheckboxListStateManagement.tsx create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss create mode 100644 frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts diff --git a/frontend/__mocks__/configMock.ts b/frontend/__mocks__/configMock.ts index 26b996662..03e555485 100644 --- a/frontend/__mocks__/configMock.ts +++ b/frontend/__mocks__/configMock.ts @@ -76,6 +76,7 @@ const DEFAULT_CONFIG_MOCK: IConfig = { integrations: { jira: [], zendesk: [], + google_calendar: [], }, logging: { debug: false, diff --git a/frontend/__mocks__/policyMock.ts b/frontend/__mocks__/policyMock.ts index 048dd6d49..c66c58a0b 100644 --- a/frontend/__mocks__/policyMock.ts +++ b/frontend/__mocks__/policyMock.ts @@ -22,6 +22,7 @@ const DEFAULT_POLICY_MOCK: IPolicyStats = { webhook: "Off", has_run: true, next_update_ms: 3600000, + calendar_events_enabled: true, }; const createMockPolicy = (overrides?: Partial): IPolicyStats => { diff --git a/frontend/components/forms/FormField/FormField.tsx b/frontend/components/forms/FormField/FormField.tsx index 68d07507a..80bf5833a 100644 --- a/frontend/components/forms/FormField/FormField.tsx +++ b/frontend/components/forms/FormField/FormField.tsx @@ -3,6 +3,7 @@ import classnames from "classnames"; import { isEmpty } from "lodash"; import TooltipWrapper from "components/TooltipWrapper"; +import { PlacesType } from "react-tooltip-5"; // all form-field styles are defined in _global.scss, which apply here and elsewhere const baseClass = "form-field"; @@ -16,6 +17,7 @@ export interface IFormFieldProps { name: string; type: string; tooltip?: React.ReactNode; + labelTooltipPosition?: PlacesType; } const FormField = ({ @@ -27,6 +29,7 @@ const FormField = ({ name, type, tooltip, + labelTooltipPosition, }: IFormFieldProps): JSX.Element => { const renderLabel = () => { const labelWrapperClasses = classnames(`${baseClass}__label`, { @@ -45,7 +48,10 @@ const FormField = ({ > {error || (tooltip ? ( - + {label as string} ) : ( diff --git a/frontend/components/forms/fields/InputField/InputField.jsx b/frontend/components/forms/fields/InputField/InputField.jsx index 83c892eb1..72969c256 100644 --- a/frontend/components/forms/fields/InputField/InputField.jsx +++ b/frontend/components/forms/fields/InputField/InputField.jsx @@ -33,6 +33,7 @@ class InputField extends Component { ]).isRequired, parseTarget: PropTypes.bool, tooltip: PropTypes.string, + labelTooltipPosition: PropTypes.string, helpText: PropTypes.oneOfType([ PropTypes.string, PropTypes.arrayOf(PropTypes.string), @@ -55,6 +56,7 @@ class InputField extends Component { value: "", parseTarget: false, tooltip: "", + labelTooltipPosition: "", helpText: "", enableCopy: false, ignore1password: false, @@ -124,6 +126,7 @@ class InputField extends Component { "error", "name", "tooltip", + "labelTooltipPosition", ]); const copyValue = (e) => { diff --git a/frontend/components/forms/fields/Slider/Slider.tsx b/frontend/components/forms/fields/Slider/Slider.tsx index 2b368275f..db21fe6c9 100644 --- a/frontend/components/forms/fields/Slider/Slider.tsx +++ b/frontend/components/forms/fields/Slider/Slider.tsx @@ -6,7 +6,10 @@ import FormField from "components/forms/FormField"; import { IFormFieldProps } from "components/forms/FormField/FormField"; interface ISliderProps { - onChange: () => void; + onChange: (newValue?: { + name: string; + value: string | number | boolean; + }) => void; value: boolean; inactiveText: string; activeText: string; diff --git a/frontend/components/graphics/CalendarEventPreview.tsx b/frontend/components/graphics/CalendarEventPreview.tsx new file mode 100644 index 000000000..4c29770a6 --- /dev/null +++ b/frontend/components/graphics/CalendarEventPreview.tsx @@ -0,0 +1,1184 @@ +import React from "react"; + +const CalendarEventPreview = () => { + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 💻 🚫  + + + + + + + + + + + + + + + + + 💻 🚫  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +}; + +export default CalendarEventPreview; diff --git a/frontend/components/graphics/index.ts b/frontend/components/graphics/index.ts index 84e10a371..fb3b0c5fd 100644 --- a/frontend/components/graphics/index.ts +++ b/frontend/components/graphics/index.ts @@ -17,6 +17,7 @@ import EmptyTeams from "./EmptyTeams"; import EmptyPacks from "./EmptyPacks"; import EmptySchedule from "./EmptySchedule"; import CollectingResults from "./CollectingResults"; +import CalendarEventPreview from "./CalendarEventPreview"; export const GRAPHIC_MAP = { // Empty state graphics @@ -41,6 +42,7 @@ export const GRAPHIC_MAP = { "file-pem": FilePem, // Other graphics "collecting-results": CollectingResults, + "calendar-event-preview": CalendarEventPreview, }; export type GraphicNames = keyof typeof GRAPHIC_MAP; diff --git a/frontend/hooks/useCheckboxListStateManagement.tsx b/frontend/hooks/useCheckboxListStateManagement.tsx new file mode 100644 index 000000000..4c1cf9d88 --- /dev/null +++ b/frontend/hooks/useCheckboxListStateManagement.tsx @@ -0,0 +1,36 @@ +import { useState } from "react"; + +import { IPolicy } from "interfaces/policy"; + +interface ICheckedPolicy { + name?: string; + id: number; + isChecked: boolean; +} + +const useCheckboxListStateManagement = ( + allPolicies: IPolicy[], + automatedPolicies: number[] | undefined +) => { + const [policyItems, setPolicyItems] = useState(() => { + return allPolicies.map(({ name, id }) => ({ + name, + id, + isChecked: !!automatedPolicies?.includes(id), + })); + }); + + const updatePolicyItems = (policyId: number) => { + setPolicyItems((prevItems) => + prevItems.map((policy) => + policy.id !== policyId + ? policy + : { ...policy, isChecked: !policy.isChecked } + ) + ); + }; + + return { policyItems, updatePolicyItems }; +}; + +export default useCheckboxListStateManagement; diff --git a/frontend/interfaces/config.ts b/frontend/interfaces/config.ts index 1df44de33..8eee167f1 100644 --- a/frontend/interfaces/config.ts +++ b/frontend/interfaces/config.ts @@ -4,7 +4,7 @@ import { IWebhookFailingPolicies, IWebhookSoftwareVulnerabilities, } from "interfaces/webhook"; -import { IIntegrations } from "./integration"; +import { IGlobalIntegrations } from "./integration"; export interface ILicense { tier: string; @@ -175,7 +175,7 @@ export interface IConfig { // databases_path: string; // }; webhook_settings: IWebhookSettings; - integrations: IIntegrations; + integrations: IGlobalIntegrations; logging: { debug: boolean; json: boolean; diff --git a/frontend/interfaces/integration.ts b/frontend/interfaces/integration.ts index adcbeeb7e..49156d627 100644 --- a/frontend/interfaces/integration.ts +++ b/frontend/interfaces/integration.ts @@ -60,7 +60,31 @@ export interface IIntegrationFormErrors { enableSoftwareVulnerabilities?: boolean; } +export interface IGlobalCalendarIntegration { + email: string; + private_key: string; + domain: string; +} + +interface ITeamCalendarSettings { + enable_calendar_events: boolean; + webhook_url: string; +} + +// zendesk and jira fields are coupled – if one is present, the other needs to be present. If +// one is present and the other is null/missing, the other will be nullified. google_calendar is +// separated – it can be present without the other 2 without nullifying them. +// TODO: Update these types to reflect this. + export interface IIntegrations { zendesk: IZendeskIntegration[]; jira: IJiraIntegration[]; } + +export interface IGlobalIntegrations extends IIntegrations { + google_calendar?: IGlobalCalendarIntegration[] | null; +} + +export interface ITeamIntegrations extends IIntegrations { + google_calendar?: ITeamCalendarSettings | null; +} diff --git a/frontend/interfaces/policy.ts b/frontend/interfaces/policy.ts index 4858de5f3..056ab7040 100644 --- a/frontend/interfaces/policy.ts +++ b/frontend/interfaces/policy.ts @@ -40,6 +40,7 @@ export interface IPolicy { created_at: string; updated_at: string; critical: boolean; + calendar_events_enabled: boolean; } // Used on the manage hosts page and other places where aggregate stats are displayed @@ -90,6 +91,7 @@ export interface IPolicyFormData { query?: string | number | boolean | undefined; team_id?: number; id?: number; + calendar_events_enabled?: boolean; } export interface IPolicyNew { diff --git a/frontend/interfaces/team.ts b/frontend/interfaces/team.ts index 435075902..8fa472602 100644 --- a/frontend/interfaces/team.ts +++ b/frontend/interfaces/team.ts @@ -1,7 +1,7 @@ import PropTypes from "prop-types"; import { IConfigFeatures, IWebhookSettings } from "./config"; import enrollSecretInterface, { IEnrollSecret } from "./enroll_secret"; -import { IIntegrations } from "./integration"; +import { ITeamIntegrations } from "./integration"; import { UserRole } from "./user"; export default PropTypes.shape({ @@ -82,7 +82,7 @@ export type ITeamWebhookSettings = Pick< */ export interface ITeamAutomationsConfig { webhook_settings: ITeamWebhookSettings; - integrations: IIntegrations; + integrations: ITeamIntegrations; } /** diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss index 1152dcfb5..06bd48a65 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss +++ b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss @@ -1,104 +1,9 @@ .host-actions-dropdown { - .form-field { - margin: 0; + @include button-dropdown; + .Select-multi-value-wrapper { + width: 55px; } - - .Select { - position: relative; - border: 0; - height: auto; - - &.is-focused, - &:hover { - border: 0; - } - - &.is-focused:not(.is-open) { - .Select-control { - background-color: initial; - } - } - - .Select-control { - display: flex; - background-color: initial; - height: auto; - justify-content: space-between; - border: 0; - cursor: pointer; - - &:hover { - box-shadow: none; - } - - &:hover .Select-placeholder { - color: $core-vibrant-blue; - } - - .Select-placeholder { - color: $core-fleet-black; - font-size: 14px; - line-height: normal; - padding-left: 0; - margin-top: 1px; - } - - .Select-input { - height: auto; - } - - .Select-arrow-zone { - display: flex; - } - } - - .Select-multi-value-wrapper { - width: 55px; - } - - .Select-placeholder { - display: flex; - align-items: center; - } - - .Select-menu-outer { - margin-top: $pad-xsmall; - box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1); - border-radius: $border-radius; - z-index: 6; - overflow: hidden; - border: 0; - width: 188px; - left: unset; - top: unset; - max-height: none; - padding: $pad-small; - position: absolute; - left: -120px; - - .Select-menu { - max-height: none; - } - } - - .Select-arrow { - transition: transform 0.25s ease; - } - - &:not(.is-open) { - .Select-control:hover .Select-arrow { - content: url("../assets/images/icon-chevron-blue-16x16@2x.png"); - } - } - - &.is-open { - .Select-control .Select-placeholder { - color: $core-vibrant-blue; - } - - .Select-arrow { - transform: rotate(180deg); - } - } + .Select > .Select-menu-outer { + left: -120px; } } diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx index 99faef93d..a117974fa 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx +++ b/frontend/pages/hosts/details/HostDetailsPage/HostDetailsPage.tsx @@ -13,7 +13,6 @@ import { QueryContext } from "context/query"; import { NotificationContext } from "context/notification"; import activitiesAPI, { - IActivitiesResponse, IPastActivitiesResponse, IUpcomingActivitiesResponse, } from "services/entities/activities"; diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx index ce359b577..80a832026 100644 --- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx @@ -2,7 +2,9 @@ import React, { useCallback, useContext, useEffect, useState } from "react"; import { useQuery } from "react-query"; import { InjectedRouter } from "react-router/lib/Router"; import PATHS from "router/paths"; -import { noop, isEqual } from "lodash"; +import { noop, isEqual, uniqueId } from "lodash"; + +import { Tooltip as ReactTooltip5 } from "react-tooltip-5"; import { getNextLocationPath } from "utilities/helpers"; @@ -34,6 +36,8 @@ import teamsAPI, { ILoadTeamResponse } from "services/entities/teams"; import { ITableQueryData } from "components/TableContainer/TableContainer"; import Button from "components/buttons/Button"; +// @ts-ignore +import Dropdown from "components/forms/fields/Dropdown"; import RevealButton from "components/buttons/RevealButton"; import Spinner from "components/Spinner"; import TeamsDropdown from "components/TeamsDropdown"; @@ -44,6 +48,8 @@ import PoliciesTable from "./components/PoliciesTable"; import OtherWorkflowsModal from "./components/OtherWorkflowsModal"; import AddPolicyModal from "./components/AddPolicyModal"; import DeletePolicyModal from "./components/DeletePolicyModal"; +import CalendarEventsModal from "./components/CalendarEventsModal"; +import { ICalendarEventsFormData } from "./components/CalendarEventsModal/CalendarEventsModal"; interface IManagePoliciesPageProps { router: InjectedRouter; @@ -125,12 +131,15 @@ const ManagePolicyPage = ({ const [isUpdatingAutomations, setIsUpdatingAutomations] = useState(false); const [isUpdatingPolicies, setIsUpdatingPolicies] = useState(false); + const [ + updatingPolicyEnabledCalendarEvents, + setUpdatingPolicyEnabledCalendarEvents, + ] = useState(false); const [selectedPolicyIds, setSelectedPolicyIds] = useState([]); - const [showManageAutomationsModal, setShowManageAutomationsModal] = useState( - false - ); + const [showOtherWorkflowsModal, setShowOtherWorkflowsModal] = useState(false); const [showAddPolicyModal, setShowAddPolicyModal] = useState(false); const [showDeletePolicyModal, setShowDeletePolicyModal] = useState(false); + const [showCalendarEventsModal, setShowCalendarEventsModal] = useState(false); const [teamPolicies, setTeamPolicies] = useState(); const [inheritedPolicies, setInheritedPolicies] = useState(); @@ -473,14 +482,30 @@ const ManagePolicyPage = ({ ] // Other dependencies can cause infinite re-renders as URL is source of truth ); - const toggleManageAutomationsModal = () => - setShowManageAutomationsModal(!showManageAutomationsModal); + const toggleOtherWorkflowsModal = () => + setShowOtherWorkflowsModal(!showOtherWorkflowsModal); const toggleAddPolicyModal = () => setShowAddPolicyModal(!showAddPolicyModal); const toggleDeletePolicyModal = () => setShowDeletePolicyModal(!showDeletePolicyModal); + const toggleCalendarEventsModal = () => { + setShowCalendarEventsModal(!showCalendarEventsModal); + }; + + const onSelectAutomationOption = (option: string) => { + switch (option) { + case "calendar_events": + toggleCalendarEventsModal(); + break; + case "other_workflows": + toggleOtherWorkflowsModal(); + break; + default: + } + }; + const toggleShowInheritedPolicies = () => { // URL source of truth const locationPath = getNextLocationPath({ @@ -496,6 +521,7 @@ const ManagePolicyPage = ({ const handleUpdateAutomations = async (requestBody: { webhook_settings: Pick; + // TODO - update below type to specify team integration integrations: IIntegrations; }) => { setIsUpdatingAutomations(true); @@ -510,13 +536,59 @@ const ManagePolicyPage = ({ "Could not update policy automations. Please try again." ); } finally { - toggleManageAutomationsModal(); + toggleOtherWorkflowsModal(); setIsUpdatingAutomations(false); refetchConfig(); isAnyTeamSelected && refetchTeamConfig(); } }; + const updatePolicyEnabledCalendarEvents = async ( + formData: ICalendarEventsFormData + ) => { + setUpdatingPolicyEnabledCalendarEvents(true); + + try { + // update enabled and URL in config + const configResponse = teamsAPI.update( + { + integrations: { + google_calendar: { + enable_calendar_events: formData.enabled, + webhook_url: formData.url, + }, + // TODO - can omit these? + zendesk: teamConfig?.integrations.zendesk || [], + jira: teamConfig?.integrations.jira || [], + }, + }, + teamIdForApi + ); + + // update policies calendar events enabled + // TODO - only update changed policies + const policyResponses = formData.policies.map((formPolicy) => + teamPoliciesAPI.update(formPolicy.id, { + calendar_events_enabled: formPolicy.isChecked, + team_id: teamIdForApi, + }) + ); + + await Promise.all([configResponse, ...policyResponses]); + renderFlash("success", "Successfully updated policy automations."); + } catch { + renderFlash( + "error", + "Could not update policy automations. Please try again." + ); + } finally { + toggleCalendarEventsModal(); + setUpdatingPolicyEnabledCalendarEvents(false); + refetchTeamPolicies(); + refetchTeamConfig(); + } + }; + const onAddPolicyClick = () => { setLastEditedQueryName(""); setLastEditedQueryDescription(""); @@ -682,6 +754,60 @@ const ManagePolicyPage = ({ ); }; + const getAutomationsDropdownOptions = () => { + const isAllTeams = teamIdForApi === undefined || teamIdForApi === -1; + let calEventsLabel: React.ReactNode = "Calendar events"; + if (!isPremiumTier) { + const tipId = uniqueId(); + calEventsLabel = ( + +
Calendar events
+ + Available in Fleet Premium + + + ); + } else if (isAllTeams) { + const tipId = uniqueId(); + calEventsLabel = ( + +
Calendar events
+ + Select a team to manage +
+ calendar events. + + + ); + } + + return [ + { + label: calEventsLabel, + value: "calendar_events", + disabled: !isPremiumTier || isAllTeams, + helpText: "Automatically reserve time to resolve failing policies.", + }, + { + label: "Other workflows", + value: "other_workflows", + disabled: false, + helpText: "Create tickets or fire webhooks for failing policies.", + }, + ]; + }; + + const isCalEventsConfigured = + (config?.integrations.google_calendar && + config?.integrations.google_calendar.length > 0) ?? + false; + return (
@@ -709,18 +835,15 @@ const ManagePolicyPage = ({ {showCtaButtons && (
{canManageAutomations && automationsConfig && ( - +
+ +
)} {canAddOrDeletePolicy && (
@@ -790,13 +913,13 @@ const ManagePolicyPage = ({ )}
)} - {config && automationsConfig && showManageAutomationsModal && ( + {config && automationsConfig && showOtherWorkflowsModal && ( )} @@ -815,6 +938,22 @@ const ManagePolicyPage = ({ onSubmit={onDeletePolicySubmit} /> )} + {showCalendarEventsModal && ( + + )}
); diff --git a/frontend/pages/policies/ManagePoliciesPage/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/_styles.scss index 62c29c1d5..ed99ad013 100644 --- a/frontend/pages/policies/ManagePoliciesPage/_styles.scss +++ b/frontend/pages/policies/ManagePoliciesPage/_styles.scss @@ -8,13 +8,33 @@ .button-wrap { display: flex; justify-content: flex-end; - min-width: 266px; + align-items: center; + gap: 8px; } } - &__manage-automations { - padding: $pad-small; - margin-right: $pad-small; + &__manage-automations-wrapper { + @include button-dropdown; + .Select-multi-value-wrapper { + width: 146px; + } + .Select > .Select-menu-outer { + left: -186px; + width: 360px; + .is-disabled * { + color: $ui-fleet-black-25; + .react-tooltip { + @include tooltip-text; + } + } + } + .Select-control { + margin-top: 0; + gap: 6px; + } + .Select-placeholder { + font-weight: $bold; + } } &__header { diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx new file mode 100644 index 000000000..eba5abb4e --- /dev/null +++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx @@ -0,0 +1,318 @@ +import React, { useCallback, useState } from "react"; + +import { IPolicy } from "interfaces/policy"; + +import validURL from "components/forms/validators/valid_url"; + +import Button from "components/buttons/Button"; +import RevealButton from "components/buttons/RevealButton"; +import CustomLink from "components/CustomLink"; +import Slider from "components/forms/fields/Slider"; +// @ts-ignore +import InputField from "components/forms/fields/InputField"; +import Graphic from "components/Graphic"; +import Modal from "components/Modal"; +import Checkbox from "components/forms/fields/Checkbox"; +import { syntaxHighlight } from "utilities/helpers"; + +const baseClass = "calendar-events-modal"; + +interface IFormPolicy { + name: string; + id: number; + isChecked: boolean; +} +export interface ICalendarEventsFormData { + enabled: boolean; + url: string; + policies: IFormPolicy[]; +} + +interface ICalendarEventsModal { + onExit: () => void; + updatePolicyEnabledCalendarEvents: ( + formData: ICalendarEventsFormData + ) => void; + isUpdating: boolean; + configured: boolean; + enabled: boolean; + url: string; + policies: IPolicy[]; +} + +// allows any policy name to be the name of a form field, one of the checkboxes +type FormNames = string; + +const CalendarEventsModal = ({ + onExit, + updatePolicyEnabledCalendarEvents, + isUpdating, + configured, + enabled, + url, + policies, +}: ICalendarEventsModal) => { + const [formData, setFormData] = useState({ + enabled, + url, + // TODO - stay udpdated on state of backend approach to syncing policies in the policies table + // and in the new calendar table + // id may change if policy was deleted + // name could change if policy was renamed + policies: policies.map((policy) => ({ + name: policy.name, + id: policy.id, + isChecked: policy.calendar_events_enabled || false, + })), + }); + const [formErrors, setFormErrors] = useState>( + {} + ); + const [showPreviewCalendarEvent, setShowPreviewCalendarEvent] = useState( + false + ); + const [showExamplePayload, setShowExamplePayload] = useState(false); + + const validateCalendarEventsFormData = ( + curFormData: ICalendarEventsFormData + ) => { + const errors: Record = {}; + if (curFormData.enabled) { + const { url: curUrl } = curFormData; + if (!validURL({ url: curUrl })) { + const errorPrefix = curUrl ? `${curUrl} is not` : "Please enter"; + errors.url = `${errorPrefix} a valid resolution webhook URL`; + } + } + return errors; + }; + + // TODO - separate change handlers for checkboxes: + // const onPolicyUpdate = ... + // const onTextFieldUpdate = ... + + const onInputChange = useCallback( + (newVal: { name: FormNames; value: string | number | boolean }) => { + const { name, value } = newVal; + let newFormData: ICalendarEventsFormData; + // for the first two fields, set the new value directly + if (["enabled", "url"].includes(name)) { + newFormData = { ...formData, [name]: value }; + } else if (typeof value === "boolean") { + // otherwise, set the value for a nested policy + const newFormPolicies = formData.policies.map((formPolicy) => { + if (formPolicy.name === name) { + return { ...formPolicy, isChecked: value }; + } + return formPolicy; + }); + newFormData = { ...formData, policies: newFormPolicies }; + } else { + throw TypeError("Unexpected value type for policy checkbox"); + } + setFormData(newFormData); + setFormErrors(validateCalendarEventsFormData(newFormData)); + }, + [formData] + ); + + const togglePreviewCalendarEvent = () => { + setShowPreviewCalendarEvent(!showPreviewCalendarEvent); + }; + + const renderExamplePayload = () => { + return ( + <> + 
POST https://server.com/example
+ 
+      
+    );
+  };
+
+  const renderPolicies = () => {
+    return (
+      

+        
Policies:

+        {formData.policies.map((policy) => {
+          const { isChecked, name, id } = policy;
+          return (
+            

+               {
+                  onInputChange({ name, value: !isChecked });
+                }}
+              >
+                {name}
+              
+            

+          );
+        })}
+        
+          A calendar event will be created for end users if one of their hosts
+          fail any of these policies.{" "}
+          
+        
+      

+    );
+  };
+  const renderPreviewCalendarEventModal = () => {
+    return (
+      
+        <>
+          
A similar event will appear in the end user's calendar:

+          
+          

+            
+          

+        
+      
+    );
+  };
+
+  const renderPlaceholderModal = () => {
+    return (
+      

+        
+          
+        
+        

+          To create calendar events for end users if their hosts fail policies,
+          you must first connect Fleet to your Google Workspace service account.
+        

+        

+          This can be configured in{" "}
+          Settings > Integrations > Calendars.
+        

+        
+        

+          
+        

+      

+    );
+  };
+
+  const renderConfiguredModal = () => (
+    

+      

+         {
+            onInputChange({ name: "enabled", value: !formData.enabled });
+          }}
+          inactiveText="Disabled"
+          activeText="Enabled"
+        />
+        
+      

+      

+        
+         {
+            setShowExamplePayload(!showExamplePayload);
+          }}
+        />
+        {showExamplePayload && renderExamplePayload()}
+        {renderPolicies()}
+      

+      

+        
+        
+      

+    

+  );
+
+  if (showPreviewCalendarEvent) {
+    return renderPreviewCalendarEventModal();
+  }
+  return (
+     {
+              updatePolicyEnabledCalendarEvents(formData);
+            }
+          : onExit
+      }
+      className={baseClass}
+      width="large"
+    >
+      {configured ? renderConfiguredModal() : renderPlaceholderModal()}
+    
+  );
+};
+
+export default CalendarEventsModal;
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss
new file mode 100644
index 000000000..3b1952a8c
--- /dev/null
+++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/_styles.scss
@@ -0,0 +1,35 @@
+.calendar-events-modal {
+  .placeholder {
+    display: flex;
+    flex-direction: column;
+    gap: 24px;
+    line-height: 150%;
+    .modal-cta-wrap {
+      margin-top: 0;
+    }
+  }
+  .form-header {
+    display: flex;
+    justify-content: space-between;
+    .button--text-link {
+      white-space: nowrap;
+    }
+  }
+
+  .form-fields {
+    &--disabled {
+      @include disabled;
+    }
+  }
+
+  pre {
+    box-sizing: border-box;
+    margin: 0;
+  }
+}
+
+.calendar-event-preview {
+  p {
+    margin: 24px 0;
+  }
+}
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts
new file mode 100644
index 000000000..b08ecf106
--- /dev/null
+++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/index.ts
@@ -0,0 +1 @@
+export { default } from "./CalendarEventsModal";
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx b/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx
index 4fc8f7973..c2c1c505a 100644
--- a/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx
+++ b/frontend/pages/policies/ManagePoliciesPage/components/PoliciesTable/PoliciesTableConfig.tsx
@@ -284,16 +284,6 @@ const generateTableHeaders = (
   ];
 
   if (tableType !== "inheritedPolicies") {
-    tableHeaders.push({
-      title: "Automations",
-      Header: "Automations",
-      disableSortBy: true,
-      accessor: "webhook",
-      Cell: (cellProps: ICellProps): JSX.Element => (
-        
-      ),
-    });
-
     if (!canAddOrDeletePolicy) {
       return tableHeaders;
     }
diff --git a/frontend/services/entities/team_policies.ts b/frontend/services/entities/team_policies.ts
index d7a03f1f1..2858938dc 100644
--- a/frontend/services/entities/team_policies.ts
+++ b/frontend/services/entities/team_policies.ts
@@ -87,6 +87,7 @@ export default {
       resolution,
       platform,
       critical,
+      calendar_events_enabled,
     } = data;
     const { TEAMS } = endpoints;
     const path = `${TEAMS}/${team_id}/policies/${id}`;
@@ -98,6 +99,7 @@ export default {
       resolution,
       platform,
       critical,
+      calendar_events_enabled,
     });
   },
   destroy: (teamId: number | undefined, ids: number[]) => {
diff --git a/frontend/services/entities/teams.ts b/frontend/services/entities/teams.ts
index 3c7e1a261..149544582 100644
--- a/frontend/services/entities/teams.ts
+++ b/frontend/services/entities/teams.ts
@@ -5,7 +5,7 @@ import { pick } from "lodash";
 
 import { buildQueryStringFromParams } from "utilities/url";
 import { IEnrollSecret } from "interfaces/enroll_secret";
-import { IIntegrations } from "interfaces/integration";
+import { ITeamIntegrations } from "interfaces/integration";
 import {
   API_NO_TEAM_ID,
   INewTeamUsersBody,
@@ -39,7 +39,7 @@ export interface ITeamFormData {
 export interface IUpdateTeamFormData {
   name: string;
   webhook_settings: Partial;
-  integrations: IIntegrations;
+  integrations: ITeamIntegrations;
   mdm: {
     macos_updates?: {
       minimum_version: string;
@@ -118,7 +118,7 @@ export default {
       requestBody.webhook_settings = webhook_settings;
     }
     if (integrations) {
-      const { jira, zendesk } = integrations;
+      const { jira, zendesk, google_calendar } = integrations;
       const teamIntegrationProps = [
         "enable_failing_policies",
         "group_id",
@@ -128,6 +128,7 @@ export default {
       requestBody.integrations = {
         jira: jira?.map((j) => pick(j, teamIntegrationProps)),
         zendesk: zendesk?.map((z) => pick(z, teamIntegrationProps)),
+        google_calendar,
       };
     }
     if (mdm) {
diff --git a/frontend/styles/var/mixins.scss b/frontend/styles/var/mixins.scss
index e786ab769..a6ef009b6 100644
--- a/frontend/styles/var/mixins.scss
+++ b/frontend/styles/var/mixins.scss
@@ -227,3 +227,103 @@ $max-width: 2560px;
   // compensate in layout for extra clickable area button height
   margin: -8px 0;
 }
+
+@mixin button-dropdown {
+  .form-field {
+    margin: 0;
+  }
+
+  .Select {
+    position: relative;
+    border: 0;
+    height: auto;
+
+    &.is-focused,
+    &:hover {
+      border: 0;
+    }
+
+    &.is-focused:not(.is-open) {
+      .Select-control {
+        background-color: initial;
+      }
+    }
+
+    .Select-control {
+      display: flex;
+      background-color: initial;
+      height: auto;
+      justify-content: space-between;
+      border: 0;
+      cursor: pointer;
+
+      &:hover {
+        box-shadow: none;
+      }
+
+      &:hover .Select-placeholder {
+        color: $core-vibrant-blue;
+      }
+
+      .Select-placeholder {
+        color: $core-fleet-black;
+        font-size: 14px;
+        line-height: normal;
+        padding-left: 0;
+        margin-top: 1px;
+      }
+
+      .Select-input {
+        height: auto;
+      }
+
+      .Select-arrow-zone {
+        display: flex;
+      }
+    }
+
+    .Select-placeholder {
+      display: flex;
+      align-items: center;
+    }
+
+    .Select-menu-outer {
+      margin-top: $pad-xsmall;
+      box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1);
+      border-radius: $border-radius;
+      z-index: 6;
+      overflow: hidden;
+      border: 0;
+      width: 188px;
+      left: unset;
+      top: unset;
+      max-height: none;
+      padding: $pad-small;
+      position: absolute;
+
+      .Select-menu {
+        max-height: none;
+      }
+    }
+
+    .Select-arrow {
+      transition: transform 0.25s ease;
+    }
+
+    &:not(.is-open) {
+      .Select-control:hover .Select-arrow {
+        content: url("../assets/images/icon-chevron-blue-16x16@2x.png");
+      }
+    }
+
+    &.is-open {
+      .Select-control .Select-placeholder {
+        color: $core-vibrant-blue;
+      }
+
+      .Select-arrow {
+        transform: rotate(180deg);
+      }
+    }
+  }
+}

From e8f177dd4368697dd5f7864cda0a07ccb2121b60 Mon Sep 17 00:00:00 2001
From: Lucas Manuel Rodriguez 
Date: Thu, 21 Mar 2024 12:23:59 -0300
Subject: [PATCH 15/36] Additional changes to happy path and cleanup cron job
 (#17757)

#17441 & #17442
---
 cmd/fleet/calendar_cron.go                    | 301 +++++++++++++-----
 cmd/fleet/calendar_cron_test.go               |  57 +++-
 server/datastore/mysql/calendar_events.go     |  92 +++++-
 .../datastore/mysql/calendar_events_test.go   | 124 +++++++-
 .../20240314085226_AddCalendarEventTables.go  |   4 +-
 server/datastore/mysql/policies.go            |   1 +
 server/datastore/mysql/schema.sql             |   3 +-
 server/fleet/calendar_events.go               |   3 +-
 server/fleet/datastore.go                     |   4 +-
 server/mock/datastore_mock.go                 |  36 ++-
 tools/webhook/README.md                       |  16 +
 tools/webhook/main.go                         |  39 +++
 12 files changed, 558 insertions(+), 122 deletions(-)
 create mode 100644 tools/webhook/README.md
 create mode 100644 tools/webhook/main.go

diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go
index e8ec7685d..fa63b487d 100644
--- a/cmd/fleet/calendar_cron.go
+++ b/cmd/fleet/calendar_cron.go
@@ -30,6 +30,12 @@ func newCalendarSchedule(
 		ctx, name, instanceID, defaultInterval, ds, ds,
 		schedule.WithAltLockID("calendar"),
 		schedule.WithLogger(logger),
+		schedule.WithJob(
+			"calendar_events_cleanup",
+			func(ctx context.Context) error {
+				return cronCalendarEventsCleanup(ctx, ds, logger)
+			},
+		),
 		schedule.WithJob(
 			"calendar_events",
 			func(ctx context.Context) error {
@@ -51,12 +57,7 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L
 		return nil
 	}
 	googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0]
-	googleCalendarConfig := calendar.GoogleCalendarConfig{
-		Context:           ctx,
-		IntegrationConfig: googleCalendarIntegrationConfig,
-		Logger:            log.With(logger, "component", "google_calendar"),
-	}
-	calendar := calendar.NewGoogleCalendar(&googleCalendarConfig)
+	calendar := createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger)
 	domain := googleCalendarIntegrationConfig.Domain
 
 	teams, err := ds.ListTeams(ctx, fleet.TeamFilter{
@@ -79,6 +80,15 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L
 	return nil
 }
 
+func createUserCalendarFromConfig(ctx context.Context, config *fleet.GoogleCalendarIntegration, logger kitlog.Logger) fleet.UserCalendar {
+	googleCalendarConfig := calendar.GoogleCalendarConfig{
+		Context:           ctx,
+		IntegrationConfig: config,
+		Logger:            log.With(logger, "component", "google_calendar"),
+	}
+	return calendar.NewGoogleCalendar(&googleCalendarConfig)
+}
+
 func cronCalendarEventsForTeam(
 	ctx context.Context,
 	ds fleet.Datastore,
@@ -110,9 +120,6 @@ func cronCalendarEventsForTeam(
 	//	- We get only one host per email that's failing policies (the one with lower host id).
 	//	- On every host, we get only the first email that matches the domain (sorted lexicographically).
 	//
-	// TODOs(lucas):
-	//	- We need to rate limit calendar requests.
-	//
 
 	policyIDs := make([]uint, 0, len(policies))
 	for _, policy := range policies {
@@ -159,15 +166,12 @@ func cronCalendarEventsForTeam(
 		level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err)
 	}
 
-	// At last we want to notify the hosts that are failing and don't have an associated email.
-	if err := fireWebhookForHostsWithoutAssociatedEmail(
-		team.Config.Integrations.GoogleCalendar.WebhookURL,
+	// At last we want to log the hosts that are failing and don't have an associated email.
+	logHostsWithoutAssociatedEmail(
 		domain,
 		failingHostsWithoutAssociatedEmail,
 		logger,
-	); err != nil {
-		level.Info(logger).Log("msg", "webhook for hosts without associated email", "err", err)
-	}
+	)
 
 	return nil
 }
@@ -182,34 +186,40 @@ func processCalendarFailingHosts(
 ) error {
 	for _, host := range hosts {
 		logger := log.With(logger, "host_id", host.HostID)
+
+		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID)
+
+		expiredEvent := false
+		webhookAlreadyFiredThisMonth := false
+		if err == nil {
+			now := time.Now()
+			webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent
+			if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) {
+				// If the webhook already fired today and the policies are still failing
+				// we give a grace period of one day for the host before we schedule a new event.
+				continue // continue with next host
+			}
+			webhookAlreadyFiredThisMonth = webhookAlreadyFired && sameMonth(now, calendarEvent.StartTime)
+			if calendarEvent.EndTime.Before(time.Now()) {
+				expiredEvent = true
+			}
+		}
+
 		if err := userCalendar.Configure(host.Email); err != nil {
 			return fmt.Errorf("configure user calendar: %w", err)
 		}
 
-		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID)
-
-		deletedExpiredEvent := false
-		if err == nil {
-			if calendarEvent.EndTime.Before(time.Now()) {
-				if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil {
-					level.Info(logger).Log("msg", "deleting existing expired calendar event", "err", err)
-					continue // continue with next host
-				}
-				deletedExpiredEvent = true
-			}
-		}
-
 		switch {
-		case err == nil && !deletedExpiredEvent:
+		case err == nil && !expiredEvent:
 			if err := processFailingHostExistingCalendarEvent(
 				ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host,
 			); err != nil {
 				level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err)
 				continue // continue with next host
 			}
-		case fleet.IsNotFound(err) || deletedExpiredEvent:
+		case fleet.IsNotFound(err) || expiredEvent:
 			if err := processFailingHostCreateCalendarEvent(
-				ctx, ds, userCalendar, orgName, host,
+				ctx, ds, userCalendar, orgName, host, webhookAlreadyFiredThisMonth,
 			); err != nil {
 				level.Info(logger).Log("msg", "process failing host create calendar event", "err", err)
 				continue // continue with next host
@@ -231,13 +241,27 @@ func processFailingHostExistingCalendarEvent(
 	calendarEvent *fleet.CalendarEvent,
 	host fleet.HostPolicyMembershipData,
 ) error {
-	updatedEvent, updated, err := calendar.GetAndUpdateEvent(
-		calendarEvent, func(bool) string {
-			return generateCalendarEventBody(orgName, host.HostDisplayName)
+	updatedEvent := calendarEvent
+	updated := false
+	now := time.Now()
+
+	// Check the user calendar every 30 minutes (and not every time)
+	// to reduce load on both Fleet and the calendar service.
+	if time.Since(calendarEvent.UpdatedAt) > 30*time.Minute {
+		var err error
+		updatedEvent, _, err = calendar.GetAndUpdateEvent(calendarEvent, func(conflict bool) string {
+			return generateCalendarEventBody(orgName, host.HostDisplayName, conflict)
 		})
-	if err != nil {
-		return fmt.Errorf("get event calendar on db: %w", err)
+		if err != nil {
+			return fmt.Errorf("get event calendar on db: %w", err)
+		}
+		// Even if fields haven't changed we want to update the calendar_events.updated_at below.
+		updated = true
+		//
+		// TODO(lucas): Check changing updatedEvent to UTC before consuming.
+		//
 	}
+
 	if updated {
 		if err := ds.UpdateCalendarEvent(ctx,
 			calendarEvent.ID,
@@ -248,16 +272,9 @@ func processFailingHostExistingCalendarEvent(
 			return fmt.Errorf("updating event calendar on db: %w", err)
 		}
 	}
-	now := time.Now()
+
 	eventInFuture := now.Before(updatedEvent.StartTime)
 	if eventInFuture {
-		// If the webhook status was sent and event was moved to the future we set the status to pending.
-		// This can happen if the admin wants to retry a remediation.
-		if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent {
-			if err := ds.UpdateHostCalendarWebhookStatus(ctx, host.HostID, fleet.CalendarWebhookStatusPending); err != nil {
-				return fmt.Errorf("update host calendar webhook status: %w", err)
-			}
-		}
 		// Nothing else to do as event is in the future.
 		return nil
 	}
@@ -297,18 +314,31 @@ func processFailingHostExistingCalendarEvent(
 	return nil
 }
 
+func sameDate(t1 time.Time, t2 time.Time) bool {
+	y1, m1, d1 := t1.Date()
+	y2, m2, d2 := t2.Date()
+	return y1 == y2 && m1 == m2 && d1 == d2
+}
+
+func sameMonth(t1 time.Time, t2 time.Time) bool {
+	y1, m1, _ := t1.Date()
+	y2, m2, _ := t2.Date()
+	return y1 == y2 && m1 == m2
+}
+
 func processFailingHostCreateCalendarEvent(
 	ctx context.Context,
 	ds fleet.Datastore,
 	userCalendar fleet.UserCalendar,
 	orgName string,
 	host fleet.HostPolicyMembershipData,
+	webhookAlreadyFiredThisMonth bool,
 ) error {
-	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar)
+	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar, webhookAlreadyFiredThisMonth)
 	if err != nil {
 		return fmt.Errorf("create event on user calendar: %w", err)
 	}
-	if _, err := ds.NewCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID); err != nil {
+	if _, err := ds.CreateOrUpdateCalendarEvent(ctx, host.Email, calendarEvent.StartTime, calendarEvent.EndTime, calendarEvent.Data, host.HostID, fleet.CalendarWebhookStatusNone); err != nil {
 		return fmt.Errorf("create calendar event on db: %w", err)
 	}
 	return nil
@@ -318,18 +348,14 @@ func attemptCreatingEventOnUserCalendar(
 	orgName string,
 	host fleet.HostPolicyMembershipData,
 	userCalendar fleet.UserCalendar,
+	webhookAlreadyFiredThisMonth bool,
 ) (*fleet.CalendarEvent, error) {
-	// TODO(lucas): Where do we handle the following case (it seems CreateEvent needs to return no slot available for the requested day if there are none or too late):
-	//
-	// - If it’s the 3rd Tuesday of the month, create an event in the upcoming slot (if available).
-	// For example, if it’s the 3rd Tuesday of the month at 10:07a, Fleet will look for an open slot starting at 10:30a.
-	// - If it’s the 3rd Tuesday, Weds, Thurs, etc. of the month and it’s past the last slot, schedule the call for the next business day.
 	year, month, today := time.Now().Date()
-	preferredDate := getPreferredCalendarEventDate(year, month, today)
+	preferredDate := getPreferredCalendarEventDate(year, month, today, webhookAlreadyFiredThisMonth)
 	for {
 		calendarEvent, err := userCalendar.CreateEvent(
-			preferredDate, func(bool) string {
-				return generateCalendarEventBody(orgName, host.HostDisplayName)
+			preferredDate, func(conflict bool) string {
+				return generateCalendarEventBody(orgName, host.HostDisplayName, conflict)
 			},
 		)
 		var dee fleet.DayEndedError
@@ -345,7 +371,10 @@ func attemptCreatingEventOnUserCalendar(
 	}
 }
 
-func getPreferredCalendarEventDate(year int, month time.Month, today int) time.Time {
+func getPreferredCalendarEventDate(
+	year int, month time.Month, today int,
+	webhookAlreadyFired bool,
+) time.Time {
 	const (
 		// 3rd Tuesday of Month
 		preferredWeekDay = time.Tuesday
@@ -360,6 +389,10 @@ func getPreferredCalendarEventDate(year int, month time.Month, today int) time.T
 	preferredDate := firstDayOfMonth.AddDate(0, 0, offset+(7*(preferredOrdinal-1)))
 	if today > preferredDate.Day() {
 		today_ := time.Date(year, month, today, 0, 0, 0, 0, time.UTC)
+		if webhookAlreadyFired {
+			nextMonth := today_.AddDate(0, 1, 0) // move to next month
+			return getPreferredCalendarEventDate(nextMonth.Year(), nextMonth.Month(), 1, false)
+		}
 		preferredDate = addBusinessDay(today_)
 	}
 	return preferredDate
@@ -379,7 +412,7 @@ func addBusinessDay(date time.Time) time.Time {
 func removeCalendarEventsFromPassingHosts(
 	ctx context.Context,
 	ds fleet.Datastore,
-	calendar fleet.UserCalendar,
+	userCalendar fleet.UserCalendar,
 	hosts []fleet.HostPolicyMembershipData,
 ) error {
 	for _, host := range hosts {
@@ -392,47 +425,42 @@ func removeCalendarEventsFromPassingHosts(
 		default:
 			return fmt.Errorf("get calendar event from DB: %w", err)
 		}
-
-		if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil {
-			return fmt.Errorf("delete db calendar event: %w", err)
-		}
-		if err := calendar.Configure(host.Email); err != nil {
-			return fmt.Errorf("connect to user calendar: %w", err)
-		}
-		if err := calendar.DeleteEvent(calendarEvent); err != nil {
-			return fmt.Errorf("delete calendar event: %w", err)
+		if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil {
+			return fmt.Errorf("delete user calendar event: %w", err)
 		}
 	}
 	return nil
 }
 
-func fireWebhookForHostsWithoutAssociatedEmail(
-	webhookURL string,
+func logHostsWithoutAssociatedEmail(
 	domain string,
 	hosts []fleet.HostPolicyMembershipData,
 	logger kitlog.Logger,
-) error {
-	// TODO(lucas): We are firing these every 5 minutes...
-	for _, host := range hosts {
-		if err := fleet.FireCalendarWebhook(
-			webhookURL,
-			host.HostID, host.HostHardwareSerial, host.HostDisplayName, nil,
-			fmt.Sprintf("No %s Google account associated with this host.", domain),
-		); err != nil {
-			level.Error(logger).Log(
-				"msg", "fire webhook for hosts without associated email", "err", err,
-			)
-		}
+) {
+	if len(hosts) == 0 {
+		return
 	}
-	return nil
+	var hostIDs []uint
+	for _, host := range hosts {
+		hostIDs = append(hostIDs, host.HostID)
+	}
+	// Logging as debug because this might get logged every 5 minutes.
+	level.Debug(logger).Log(
+		"msg", fmt.Sprintf("no %s Google account associated with the hosts", domain),
+		"host_ids", fmt.Sprintf("%+v", hostIDs),
+	)
 }
 
-func generateCalendarEventBody(orgName, hostDisplayName string) string {
+func generateCalendarEventBody(orgName, hostDisplayName string, conflict bool) string {
+	conflictStr := ""
+	if conflict {
+		conflictStr = " because there was no remaining availability"
+	}
 	return fmt.Sprintf(`Please leave your computer on and connected to power.
 
 Expect an automated restart.
 
-%s reserved this time to fix %s.`, orgName, hostDisplayName,
+%s reserved this time to fix %s%s.`, orgName, hostDisplayName, conflictStr,
 	)
 }
 
@@ -456,3 +484,112 @@ func isHostOnline(ctx context.Context, ds fleet.Datastore, hostID uint) (bool, e
 		return false, fmt.Errorf("unknown host status: %s", status)
 	}
 }
+
+func cronCalendarEventsCleanup(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger) error {
+	appConfig, err := ds.AppConfig(ctx)
+	if err != nil {
+		return fmt.Errorf("load app config: %w", err)
+	}
+
+	var userCalendar fleet.UserCalendar
+	if len(appConfig.Integrations.GoogleCalendar) > 0 {
+		googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0]
+		userCalendar = createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger)
+	}
+
+	// If global setting is disabled, we remove all calendar events from the DB
+	// (we cannot delete the events from the user calendar because there's no configuration anymore).
+	if userCalendar == nil {
+		if err := deleteAllCalendarEvents(ctx, ds, nil, nil); err != nil {
+			return fmt.Errorf("delete all calendar events: %w", err)
+		}
+		// We've deleted all calendar events, nothing else to do.
+		return nil
+	}
+
+	//
+	// Feature is configured globally, but now we have to check team by team.
+	//
+
+	teams, err := ds.ListTeams(ctx, fleet.TeamFilter{
+		User: &fleet.User{
+			GlobalRole: ptr.String(fleet.RoleAdmin),
+		},
+	}, fleet.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("list teams: %w", err)
+	}
+
+	for _, team := range teams {
+		if err := deleteTeamCalendarEvents(ctx, ds, userCalendar, *team); err != nil {
+			level.Info(logger).Log("msg", "delete team calendar events", "team_id", team.ID, "err", err)
+		}
+	}
+
+	//
+	// Delete calendar events from DB that haven't been updated for a while
+	// (e.g. host was transferred to another team or global).
+	//
+
+	outOfDateCalendarEvents, err := ds.ListOutOfDateCalendarEvents(ctx, time.Now().Add(-48*time.Hour))
+	if err != nil {
+		return fmt.Errorf("list out of date calendar events: %w", err)
+	}
+	for _, outOfDateCalendarEvent := range outOfDateCalendarEvents {
+		if err := deleteCalendarEvent(ctx, ds, userCalendar, outOfDateCalendarEvent); err != nil {
+			return fmt.Errorf("delete user calendar event: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func deleteAllCalendarEvents(
+	ctx context.Context,
+	ds fleet.Datastore,
+	userCalendar fleet.UserCalendar,
+	teamID *uint,
+) error {
+	calendarEvents, err := ds.ListCalendarEvents(ctx, teamID)
+	if err != nil {
+		return fmt.Errorf("list calendar events: %w", err)
+	}
+	for _, calendarEvent := range calendarEvents {
+		if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil {
+			return fmt.Errorf("delete user calendar event: %w", err)
+		}
+	}
+	return nil
+}
+
+func deleteTeamCalendarEvents(
+	ctx context.Context,
+	ds fleet.Datastore,
+	userCalendar fleet.UserCalendar,
+	team fleet.Team,
+) error {
+	if team.Config.Integrations.GoogleCalendar != nil &&
+		team.Config.Integrations.GoogleCalendar.Enable {
+		// Feature is enabled, nothing to cleanup.
+		return nil
+	}
+	return deleteAllCalendarEvents(ctx, ds, userCalendar, &team.ID)
+}
+
+func deleteCalendarEvent(ctx context.Context, ds fleet.Datastore, userCalendar fleet.UserCalendar, calendarEvent *fleet.CalendarEvent) error {
+	if userCalendar != nil {
+		// Only delete events from the user's calendar if the event is in the future.
+		if eventInFuture := time.Now().Before(calendarEvent.StartTime); eventInFuture {
+			if err := userCalendar.Configure(calendarEvent.Email); err != nil {
+				return fmt.Errorf("connect to user calendar: %w", err)
+			}
+			if err := userCalendar.DeleteEvent(calendarEvent); err != nil {
+				return fmt.Errorf("delete calendar event: %w", err)
+			}
+		}
+	}
+	if err := ds.DeleteCalendarEvent(ctx, calendarEvent.ID); err != nil {
+		return fmt.Errorf("delete db calendar event: %w", err)
+	}
+	return nil
+}
diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go
index 680cf50d9..905b79c87 100644
--- a/cmd/fleet/calendar_cron_test.go
+++ b/cmd/fleet/calendar_cron_test.go
@@ -12,34 +12,61 @@ func TestGetPreferredCalendarEventDate(t *testing.T) {
 		return time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
 	}
 	for _, tc := range []struct {
-		name  string
-		year  int
-		month time.Month
-		days  int
+		name                  string
+		year                  int
+		month                 time.Month
+		daysStart             int
+		daysEnd               int
+		webhookFiredThisMonth bool
 
 		expected time.Time
 	}{
 		{
-			year:     2024,
-			month:    3,
-			days:     31,
-			name:     "March 2024",
+			name:                  "March 2024 (webhook hasn't fired)",
+			year:                  2024,
+			month:                 3,
+			daysStart:             1,
+			daysEnd:               31,
+			webhookFiredThisMonth: false,
+
 			expected: date(2024, 3, 19),
 		},
 		{
-			year:     2024,
-			month:    4,
-			days:     30,
-			name:     "April 2024",
+			name:                  "March 2024 (webhook has fired, days before 3rd Tuesday)",
+			year:                  2024,
+			month:                 3,
+			daysStart:             1,
+			daysEnd:               18,
+			webhookFiredThisMonth: true,
+
+			expected: date(2024, 3, 19),
+		},
+		{
+			name:                  "March 2024 (webhook has fired, days after 3rd Tuesday)",
+			year:                  2024,
+			month:                 3,
+			daysStart:             20,
+			daysEnd:               30,
+			webhookFiredThisMonth: true,
+
+			expected: date(2024, 4, 16),
+		},
+		{
+			name:                  "April 2024 (webhook hasn't fired)",
+			year:                  2024,
+			month:                 4,
+			daysEnd:               30,
+			webhookFiredThisMonth: false,
+
 			expected: date(2024, 4, 16),
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			for day := 1; day <= tc.days; day++ {
-				actual := getPreferredCalendarEventDate(tc.year, tc.month, day)
+			for day := tc.daysStart; day <= tc.daysEnd; day++ {
+				actual := getPreferredCalendarEventDate(tc.year, tc.month, day, tc.webhookFiredThisMonth)
 				require.NotEqual(t, actual.Weekday(), time.Saturday)
 				require.NotEqual(t, actual.Weekday(), time.Sunday)
-				if day <= tc.expected.Day() {
+				if day <= tc.expected.Day() || tc.webhookFiredThisMonth {
 					require.Equal(t, tc.expected, actual)
 				} else {
 					today := date(tc.year, tc.month, day)
diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go
index 399091597..5ffc0f77f 100644
--- a/server/datastore/mysql/calendar_events.go
+++ b/server/datastore/mysql/calendar_events.go
@@ -11,15 +11,16 @@ import (
 	"github.com/jmoiron/sqlx"
 )
 
-func (ds *Datastore) NewCalendarEvent(
+func (ds *Datastore) CreateOrUpdateCalendarEvent(
 	ctx context.Context,
 	email string,
 	startTime time.Time,
 	endTime time.Time,
 	data []byte,
 	hostID uint,
+	webhookStatus fleet.CalendarWebhookStatus,
 ) (*fleet.CalendarEvent, error) {
-	var calendarEvent *fleet.CalendarEvent
+	var id int64
 	if err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
 		const calendarEventsQuery = `
 			INSERT INTO calendar_events (
@@ -27,7 +28,12 @@ func (ds *Datastore) NewCalendarEvent(
 				start_time,
 				end_time,
 				event
-			) VALUES (?, ?, ?, ?);
+			) VALUES (?, ?, ?, ?)
+			ON DUPLICATE KEY UPDATE
+				start_time = VALUES(start_time),
+				end_time = VALUES(end_time),
+				event = VALUES(event),
+				updated_at = CURRENT_TIMESTAMP;
 		`
 		result, err := tx.ExecContext(
 			ctx,
@@ -41,13 +47,13 @@ func (ds *Datastore) NewCalendarEvent(
 			return ctxerr.Wrap(ctx, err, "insert calendar event")
 		}
 
-		id, _ := result.LastInsertId()
-		calendarEvent = &fleet.CalendarEvent{
-			ID:        uint(id),
-			Email:     email,
-			StartTime: startTime,
-			EndTime:   endTime,
-			Data:      data,
+		if insertOnDuplicateDidInsert(result) {
+			id, _ = result.LastInsertId()
+		} else {
+			stmt := `SELECT id FROM calendar_events WHERE email = ?`
+			if err := sqlx.GetContext(ctx, tx, &id, stmt, email); err != nil {
+				return ctxerr.Wrap(ctx, err, "query mdm solution id")
+			}
 		}
 
 		const hostCalendarEventsQuery = `
@@ -55,14 +61,17 @@ func (ds *Datastore) NewCalendarEvent(
 				host_id,
 				calendar_event_id,
 				webhook_status
-			) VALUES (?, ?, ?);
+			) VALUES (?, ?, ?)
+			ON DUPLICATE KEY UPDATE
+				webhook_status = VALUES(webhook_status),
+				calendar_event_id = VALUES(calendar_event_id);
 		`
 		result, err = tx.ExecContext(
 			ctx,
 			hostCalendarEventsQuery,
 			hostID,
-			calendarEvent.ID,
-			fleet.CalendarWebhookStatusPending,
+			id,
+			webhookStatus,
 		)
 		if err != nil {
 			return ctxerr.Wrap(ctx, err, "insert host calendar event")
@@ -71,9 +80,29 @@ func (ds *Datastore) NewCalendarEvent(
 	}); err != nil {
 		return nil, ctxerr.Wrap(ctx, err)
 	}
+
+	calendarEvent, err := getCalendarEventByID(ctx, ds.writer(ctx), uint(id))
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "get created calendar event by id")
+	}
 	return calendarEvent, nil
 }
 
+func getCalendarEventByID(ctx context.Context, q sqlx.QueryerContext, id uint) (*fleet.CalendarEvent, error) {
+	const calendarEventsQuery = `
+		SELECT * FROM calendar_events WHERE id = ?;
+	`
+	var calendarEvent fleet.CalendarEvent
+	err := sqlx.GetContext(ctx, q, &calendarEvent, calendarEventsQuery, id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithID(id))
+		}
+		return nil, ctxerr.Wrap(ctx, err, "get calendar event")
+	}
+	return &calendarEvent, nil
+}
+
 func (ds *Datastore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) {
 	const calendarEventsQuery = `
 		SELECT * FROM calendar_events WHERE email = ?;
@@ -94,7 +123,8 @@ func (ds *Datastore) UpdateCalendarEvent(ctx context.Context, calendarEventID ui
 		UPDATE calendar_events SET
 			start_time = ?,
 			end_time = ?,
-			event = ?
+			event = ?,
+			updated_at = CURRENT_TIMESTAMP
 		WHERE id = ?;
 	`
 	if _, err := ds.writer(ctx).ExecContext(ctx, calendarEventsQuery, startTime, endTime, data, calendarEventID); err != nil {
@@ -148,3 +178,37 @@ func (ds *Datastore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID
 	}
 	return nil
 }
+
+func (ds *Datastore) ListCalendarEvents(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error) {
+	calendarEventsQuery := `
+		SELECT ce.* FROM calendar_events ce
+	`
+
+	var args []interface{}
+	if teamID != nil {
+		// TODO(lucas): Should we add a team_id column to calendar_events?
+		calendarEventsQuery += ` JOIN host_calendar_events hce ON ce.id=hce.calendar_event_id
+								 JOIN hosts h ON h.id=hce.host_id WHERE h.team_id = ?`
+		args = append(args, *teamID)
+	}
+
+	var calendarEvents []*fleet.CalendarEvent
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &calendarEvents, calendarEventsQuery, args...); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, ctxerr.Wrap(ctx, err, "get all calendar events")
+	}
+	return calendarEvents, nil
+}
+
+func (ds *Datastore) ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error) {
+	calendarEventsQuery := `
+		SELECT ce.* FROM calendar_events ce WHERE updated_at < ?
+	`
+	var calendarEvents []*fleet.CalendarEvent
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &calendarEvents, calendarEventsQuery, t); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "get all calendar events")
+	}
+	return calendarEvents, nil
+}
diff --git a/server/datastore/mysql/calendar_events_test.go b/server/datastore/mysql/calendar_events_test.go
index ccf07b3c7..3c6030adf 100644
--- a/server/datastore/mysql/calendar_events_test.go
+++ b/server/datastore/mysql/calendar_events_test.go
@@ -1,6 +1,128 @@
 package mysql
 
-import "testing"
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/ptr"
+	"github.com/stretchr/testify/require"
+)
 
 func TestCalendarEvents(t *testing.T) {
+	ds := CreateMySQLDS(t)
+
+	cases := []struct {
+		name string
+		fn   func(t *testing.T, ds *Datastore)
+	}{
+		{"UpdateCalendarEvent", testUpdateCalendarEvent},
+		{"CreateOrUpdateCalendarEvent", testCreateOrUpdateCalendarEvent},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			defer TruncateTables(t, ds)
+
+			c.fn(t, ds)
+		})
+	}
+}
+
+func testUpdateCalendarEvent(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+
+	host, err := ds.NewHost(context.Background(), &fleet.Host{
+		DetailUpdatedAt: time.Now(),
+		LabelUpdatedAt:  time.Now(),
+		PolicyUpdatedAt: time.Now(),
+		SeenTime:        time.Now(),
+		NodeKey:         ptr.String("1"),
+		UUID:            "1",
+		Hostname:        "foo.local",
+		PrimaryIP:       "192.168.1.1",
+		PrimaryMac:      "30-65-EC-6F-C4-58",
+	})
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host.ID, []*fleet.HostDeviceMapping{
+		{
+			HostID: host.ID,
+			Email:  "foo@example.com",
+			Source: "google_chrome_profiles",
+		},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+
+	startTime1 := time.Now()
+	endTime1 := startTime1.Add(30 * time.Minute)
+	calendarEvent, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone)
+	require.NoError(t, err)
+
+	time.Sleep(1 * time.Second)
+
+	err = ds.UpdateCalendarEvent(ctx, calendarEvent.ID, startTime1, endTime1, []byte(`{}`))
+	require.NoError(t, err)
+
+	calendarEvent2, err := ds.GetCalendarEvent(ctx, "foo@example.com")
+	require.NoError(t, err)
+	require.NotEqual(t, *calendarEvent, *calendarEvent2)
+	calendarEvent.UpdatedAt = calendarEvent2.UpdatedAt
+	require.Equal(t, *calendarEvent, *calendarEvent2)
+
+	// TODO(lucas): Add more tests here.
+}
+
+func testCreateOrUpdateCalendarEvent(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+
+	host, err := ds.NewHost(context.Background(), &fleet.Host{
+		DetailUpdatedAt: time.Now(),
+		LabelUpdatedAt:  time.Now(),
+		PolicyUpdatedAt: time.Now(),
+		SeenTime:        time.Now(),
+		NodeKey:         ptr.String("1"),
+		UUID:            "1",
+		Hostname:        "foo.local",
+		PrimaryIP:       "192.168.1.1",
+		PrimaryMac:      "30-65-EC-6F-C4-58",
+	})
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host.ID, []*fleet.HostDeviceMapping{
+		{
+			HostID: host.ID,
+			Email:  "foo@example.com",
+			Source: "google_chrome_profiles",
+		},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+
+	startTime1 := time.Now()
+	endTime1 := startTime1.Add(30 * time.Minute)
+	calendarEvent, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone)
+	require.NoError(t, err)
+
+	time.Sleep(1 * time.Second)
+
+	calendarEvent2, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime1, endTime1, []byte(`{}`), host.ID, fleet.CalendarWebhookStatusNone)
+	require.NoError(t, err)
+	require.Greater(t, calendarEvent2.UpdatedAt, calendarEvent.UpdatedAt)
+	calendarEvent.UpdatedAt = calendarEvent2.UpdatedAt
+	require.Equal(t, *calendarEvent, *calendarEvent2)
+
+	time.Sleep(1 * time.Second)
+
+	startTime2 := startTime1.Add(1 * time.Hour)
+	endTime2 := startTime1.Add(30 * time.Minute)
+	calendarEvent3, err := ds.CreateOrUpdateCalendarEvent(ctx, "foo@example.com", startTime2, endTime2, []byte(`{"foo": "bar"}`), host.ID, fleet.CalendarWebhookStatusPending)
+	require.NoError(t, err)
+	require.Greater(t, calendarEvent3.UpdatedAt, calendarEvent2.UpdatedAt)
+	require.WithinDuration(t, startTime2, calendarEvent3.StartTime, 1*time.Second)
+	require.WithinDuration(t, endTime2, calendarEvent3.EndTime, 1*time.Second)
+	require.Equal(t, string(calendarEvent3.Data), `{"foo": "bar"}`)
+
+	calendarEvent3b, err := ds.GetCalendarEvent(ctx, "foo@example.com")
+	require.NoError(t, err)
+	require.Equal(t, calendarEvent3, calendarEvent3b)
+
+	// TODO(lucas): Add more tests here.
 }
diff --git a/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go
index e9222e9d9..a385a1da8 100644
--- a/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go
+++ b/server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go
@@ -21,7 +21,9 @@ func Up_20240314085226(tx *sql.Tx) error {
 		event JSON NOT NULL,
 
 		created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-		updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+		updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+		UNIQUE KEY idx_one_calendar_event_per_email (email)
 	);
 `); err != nil {
 		return fmt.Errorf("create calendar_events table: %w", err)
diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go
index 0b3498319..b711c8b93 100644
--- a/server/datastore/mysql/policies.go
+++ b/server/datastore/mysql/policies.go
@@ -1172,6 +1172,7 @@ func (ds *Datastore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fl
 }
 
 // TODO(lucas): Must be tested at scale.
+// TODO(lucas): Filter out hosts with team_id == NULL
 func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
 	query := `
 	SELECT 
diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql
index a256591e2..cc3c88c7e 100644
--- a/server/datastore/mysql/schema.sql
+++ b/server/datastore/mysql/schema.sql
@@ -52,7 +52,8 @@ CREATE TABLE `calendar_events` (
   `event` json NOT NULL,
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-  PRIMARY KEY (`id`)
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `idx_one_calendar_event_per_email` (`email`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
diff --git a/server/fleet/calendar_events.go b/server/fleet/calendar_events.go
index 348cb074a..3152ee65b 100644
--- a/server/fleet/calendar_events.go
+++ b/server/fleet/calendar_events.go
@@ -15,7 +15,8 @@ type CalendarEvent struct {
 type CalendarWebhookStatus int
 
 const (
-	CalendarWebhookStatusPending CalendarWebhookStatus = iota
+	CalendarWebhookStatusNone CalendarWebhookStatus = iota
+	CalendarWebhookStatusPending
 	CalendarWebhookStatusSent
 )
 
diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go
index f2178bf32..65098efdf 100644
--- a/server/fleet/datastore.go
+++ b/server/fleet/datastore.go
@@ -619,12 +619,14 @@ type Datastore interface {
 	///////////////////////////////////////////////////////////////////////////////
 	// Calendar events
 
-	NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*CalendarEvent, error)
+	CreateOrUpdateCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus CalendarWebhookStatus) (*CalendarEvent, error)
 	GetCalendarEvent(ctx context.Context, email string) (*CalendarEvent, error)
 	DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error
 	UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error
 	GetHostCalendarEvent(ctx context.Context, hostID uint) (*HostCalendarEvent, *CalendarEvent, error)
 	UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status CalendarWebhookStatus) error
+	ListCalendarEvents(ctx context.Context, teamID *uint) ([]*CalendarEvent, error)
+	ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*CalendarEvent, error)
 
 	///////////////////////////////////////////////////////////////////////////////
 	// Team Policies
diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go
index 4e35d1eef..1b77b29cb 100644
--- a/server/mock/datastore_mock.go
+++ b/server/mock/datastore_mock.go
@@ -462,7 +462,7 @@ type DeleteSoftwareVulnerabilitiesFunc func(ctx context.Context, vulnerabilities
 
 type DeleteOutOfDateVulnerabilitiesFunc func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error
 
-type NewCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error)
+type CreateOrUpdateCalendarEventFunc func(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus fleet.CalendarWebhookStatus) (*fleet.CalendarEvent, error)
 
 type GetCalendarEventFunc func(ctx context.Context, email string) (*fleet.CalendarEvent, error)
 
@@ -474,6 +474,10 @@ type GetHostCalendarEventFunc func(ctx context.Context, hostID uint) (*fleet.Hos
 
 type UpdateHostCalendarWebhookStatusFunc func(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error
 
+type ListCalendarEventsFunc func(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error)
+
+type ListOutOfDateCalendarEventsFunc func(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error)
+
 type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error)
 
 type ListTeamPoliciesFunc func(ctx context.Context, teamID uint, opts fleet.ListOptions, iopts fleet.ListOptions) (teamPolicies []*fleet.Policy, inheritedPolicies []*fleet.Policy, err error)
@@ -1541,8 +1545,8 @@ type DataStore struct {
 	DeleteOutOfDateVulnerabilitiesFunc        DeleteOutOfDateVulnerabilitiesFunc
 	DeleteOutOfDateVulnerabilitiesFuncInvoked bool
 
-	NewCalendarEventFunc        NewCalendarEventFunc
-	NewCalendarEventFuncInvoked bool
+	CreateOrUpdateCalendarEventFunc        CreateOrUpdateCalendarEventFunc
+	CreateOrUpdateCalendarEventFuncInvoked bool
 
 	GetCalendarEventFunc        GetCalendarEventFunc
 	GetCalendarEventFuncInvoked bool
@@ -1559,6 +1563,12 @@ type DataStore struct {
 	UpdateHostCalendarWebhookStatusFunc        UpdateHostCalendarWebhookStatusFunc
 	UpdateHostCalendarWebhookStatusFuncInvoked bool
 
+	ListCalendarEventsFunc        ListCalendarEventsFunc
+	ListCalendarEventsFuncInvoked bool
+
+	ListOutOfDateCalendarEventsFunc        ListOutOfDateCalendarEventsFunc
+	ListOutOfDateCalendarEventsFuncInvoked bool
+
 	NewTeamPolicyFunc        NewTeamPolicyFunc
 	NewTeamPolicyFuncInvoked bool
 
@@ -3716,11 +3726,11 @@ func (s *DataStore) DeleteOutOfDateVulnerabilities(ctx context.Context, source f
 	return s.DeleteOutOfDateVulnerabilitiesFunc(ctx, source, duration)
 }
 
-func (s *DataStore) NewCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint) (*fleet.CalendarEvent, error) {
+func (s *DataStore) CreateOrUpdateCalendarEvent(ctx context.Context, email string, startTime time.Time, endTime time.Time, data []byte, hostID uint, webhookStatus fleet.CalendarWebhookStatus) (*fleet.CalendarEvent, error) {
 	s.mu.Lock()
-	s.NewCalendarEventFuncInvoked = true
+	s.CreateOrUpdateCalendarEventFuncInvoked = true
 	s.mu.Unlock()
-	return s.NewCalendarEventFunc(ctx, email, startTime, endTime, data, hostID)
+	return s.CreateOrUpdateCalendarEventFunc(ctx, email, startTime, endTime, data, hostID, webhookStatus)
 }
 
 func (s *DataStore) GetCalendarEvent(ctx context.Context, email string) (*fleet.CalendarEvent, error) {
@@ -3758,6 +3768,20 @@ func (s *DataStore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID
 	return s.UpdateHostCalendarWebhookStatusFunc(ctx, hostID, status)
 }
 
+func (s *DataStore) ListCalendarEvents(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error) {
+	s.mu.Lock()
+	s.ListCalendarEventsFuncInvoked = true
+	s.mu.Unlock()
+	return s.ListCalendarEventsFunc(ctx, teamID)
+}
+
+func (s *DataStore) ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*fleet.CalendarEvent, error) {
+	s.mu.Lock()
+	s.ListOutOfDateCalendarEventsFuncInvoked = true
+	s.mu.Unlock()
+	return s.ListOutOfDateCalendarEventsFunc(ctx, t)
+}
+
 func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) {
 	s.mu.Lock()
 	s.NewTeamPolicyFuncInvoked = true
diff --git a/tools/webhook/README.md b/tools/webhook/README.md
new file mode 100644
index 000000000..6bdb2416a
--- /dev/null
+++ b/tools/webhook/README.md
@@ -0,0 +1,16 @@
+# webhook
+
+Test tool for Fleet features that use webhook URLs.
+It reads and parses the request a JSON body and prints the JSON to standard output (with indentation).
+
+```sh
+go run ./tools/webhook 8082
+2024/03/20 09:10:00 {
+  "error": "No fleetdm.com Google account associated with this host.",
+  "host_display_name": "dChYnk.uxURT",
+  "host_id": 2,
+  "host_serial_number": "",
+  "timestamp": "2024-03-20T09:10:00.129982-03:00"
+}
+...
+```
diff --git a/tools/webhook/main.go b/tools/webhook/main.go
new file mode 100644
index 000000000..452e0a15c
--- /dev/null
+++ b/tools/webhook/main.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"os"
+)
+
+func main() {
+	log.SetFlags(log.LstdFlags)
+
+	http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			log.Printf("failed to read body: %s", err)
+			return
+		}
+
+		var v interface{}
+		if err := json.Unmarshal(body, &v); err != nil {
+			log.Printf("failed to parse JSON body: %s", err)
+			return
+		}
+		b, err := json.MarshalIndent(v, "", "  ")
+		if err != nil {
+			panic(err)
+		}
+		log.Printf("%s", b)
+
+		w.WriteHeader(http.StatusOK)
+	}))
+	//nolint:gosec // G114: file server used for testing purposes only.
+	err := http.ListenAndServe("0.0.0.0:"+os.Args[1], nil)
+	if err != nil {
+		panic(err)
+	}
+}

From 2940b32a06ec1c26049f7447e5f3efff476a59d1 Mon Sep 17 00:00:00 2001
From: RachelElysia <71795832+RachelElysia@users.noreply.github.com>
Date: Thu, 21 Mar 2024 16:22:47 -0400
Subject: [PATCH 16/36] Fleet UI: Calendar settings page (#17593)

---
 .../PlatformWrapper/_styles.scss              |  12 +-
 .../EnrollSecretRow/_styles.scss              |   5 +-
 .../InputFieldHiddenContent/_styles.scss      |   5 +-
 frontend/interfaces/integration.ts            |   3 +-
 .../TokenSecretField/_styles.scss             |   7 +-
 .../ManageSoftwareAutomationsModal.tsx        |   4 +-
 .../IntegrationsPage/IntegrationNavItems.tsx  |  56 +--
 .../IntegrationsPage/IntegrationsPage.tsx     |   8 +-
 .../cards/Calendars/Calendars.tsx             | 401 ++++++++++++++++++
 .../cards/Calendars/_styles.scss              |  57 +++
 .../IntegrationsPage/cards/Calendars/index.ts |   1 +
 .../cards/Integrations/Integrations.tsx       |  14 +-
 .../admin/components/SideNav/SideNav.tsx      |   1 -
 .../OtherWorkflowsModal.tsx                   |  12 +-
 frontend/router/paths.ts                      |   1 +
 frontend/styles/var/mixins.scss               |   9 +
 16 files changed, 528 insertions(+), 68 deletions(-)
 create mode 100644 frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
 create mode 100644 frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss
 create mode 100644 frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts

diff --git a/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss b/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss
index 7182a6968..6af47c2b7 100644
--- a/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss
+++ b/frontend/components/AddHostsModal/PlatformWrapper/_styles.scss
@@ -51,12 +51,7 @@
   }
 
   &__copy-message {
-    font-weight: $regular;
-    vertical-align: top;
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 
   .buttons {
@@ -122,9 +117,6 @@
   }
 
   &__copy-message {
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 }
diff --git a/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss b/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss
index c400a9f65..1b5d033a5 100644
--- a/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss
+++ b/frontend/components/EnrollSecrets/EnrollSecretTable/EnrollSecretRow/_styles.scss
@@ -40,10 +40,7 @@
   }
 
   &__copy-message {
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 
   &__action-overlay {
diff --git a/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss b/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss
index 89b07fdaf..4ac1e32a4 100644
--- a/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss
+++ b/frontend/components/forms/fields/InputFieldHiddenContent/_styles.scss
@@ -43,9 +43,6 @@
   }
 
   &__copy-message {
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 }
diff --git a/frontend/interfaces/integration.ts b/frontend/interfaces/integration.ts
index 49156d627..aea79f99e 100644
--- a/frontend/interfaces/integration.ts
+++ b/frontend/interfaces/integration.ts
@@ -61,9 +61,8 @@ export interface IIntegrationFormErrors {
 }
 
 export interface IGlobalCalendarIntegration {
-  email: string;
-  private_key: string;
   domain: string;
+  api_key_json: string;
 }
 
 interface ITeamCalendarSettings {
diff --git a/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss b/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss
index fee0ee67c..bca40979d 100644
--- a/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss
+++ b/frontend/pages/AccountPage/APITokenModal/TokenSecretField/_styles.scss
@@ -31,12 +31,7 @@
   }
 
   &__copy-message {
-    font-weight: $regular;
-    vertical-align: top;
-    background-color: $ui-light-grey;
-    border: solid 1px #e2e4ea;
-    border-radius: 10px;
-    padding: 2px 6px;
+    @include copy-message;
   }
 
   &__secret-download-icon {
diff --git a/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx b/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx
index 002bca892..2869dd799 100644
--- a/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx
+++ b/frontend/pages/SoftwarePage/components/ManageSoftwareAutomationsModal/ManageSoftwareAutomationsModal.tsx
@@ -8,7 +8,7 @@ import {
   IJiraIntegration,
   IZendeskIntegration,
   IIntegration,
-  IIntegrations,
+  IGlobalIntegrations,
   IIntegrationType,
 } from "interfaces/integration";
 import {
@@ -124,7 +124,7 @@ const ManageAutomationsModal = ({
     }
   }, [destinationUrl]);
 
-  const { data: integrations } = useQuery(
+  const { data: integrations } = useQuery(
     ["integrations"],
     () => configAPI.loadAll(),
     {
diff --git a/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx b/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx
index 870444974..a3f8734da 100644
--- a/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx
+++ b/frontend/pages/admin/IntegrationsPage/IntegrationNavItems.tsx
@@ -4,32 +4,34 @@ import { ISideNavItem } from "../components/SideNav/SideNav";
 import Integrations from "./cards/Integrations";
 import Mdm from "./cards/MdmSettings/MdmSettings";
 import AutomaticEnrollment from "./cards/AutomaticEnrollment/AutomaticEnrollment";
+import Calendars from "./cards/Calendars/Calendars";
 
-const getFilteredIntegrationSettingsNavItems = (
-  isSandboxMode = false
-): ISideNavItem[] => {
-  return [
-    // TODO: types
-    {
-      title: "Ticket destinations",
-      urlSection: "ticket-destinations",
-      path: PATHS.ADMIN_INTEGRATIONS_TICKET_DESTINATIONS,
-      Card: Integrations,
-    },
-    {
-      title: "Mobile device management (MDM)",
-      urlSection: "mdm",
-      path: PATHS.ADMIN_INTEGRATIONS_MDM,
-      Card: Mdm,
-      exclude: isSandboxMode,
-    },
-    {
-      title: "Automatic enrollment",
-      urlSection: "automatic-enrollment",
-      path: PATHS.ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT,
-      Card: AutomaticEnrollment,
-    },
-  ].filter((navItem) => !navItem.exclude);
-};
+const integrationSettingsNavItems: ISideNavItem[] = [
+  // TODO: types
+  {
+    title: "Ticket destinations",
+    urlSection: "ticket-destinations",
+    path: PATHS.ADMIN_INTEGRATIONS_TICKET_DESTINATIONS,
+    Card: Integrations,
+  },
+  {
+    title: "Mobile device management (MDM)",
+    urlSection: "mdm",
+    path: PATHS.ADMIN_INTEGRATIONS_MDM,
+    Card: Mdm,
+  },
+  {
+    title: "Automatic enrollment",
+    urlSection: "automatic-enrollment",
+    path: PATHS.ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT,
+    Card: AutomaticEnrollment,
+  },
+  {
+    title: "Calendars",
+    urlSection: "calendars",
+    path: PATHS.ADMIN_INTEGRATIONS_CALENDARS,
+    Card: Calendars,
+  },
+];
 
-export default getFilteredIntegrationSettingsNavItems;
+export default integrationSettingsNavItems;
diff --git a/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx b/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx
index 019a219d5..bae02c33c 100644
--- a/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx
+++ b/frontend/pages/admin/IntegrationsPage/IntegrationsPage.tsx
@@ -1,9 +1,8 @@
-import { AppContext } from "context/app";
-import React, { useContext } from "react";
+import React from "react";
 import { InjectedRouter, Params } from "react-router/lib/Router";
 
 import SideNav from "../components/SideNav";
-import getFilteredIntegrationSettingsNavItems from "./IntegrationNavItems";
+import integrationSettingsNavItems from "./IntegrationNavItems";
 
 const baseClass = "integrations";
 
@@ -16,9 +15,8 @@ const IntegrationsPage = ({
   router,
   params,
 }: IIntegrationSettingsPageProps) => {
-  const { isSandboxMode } = useContext(AppContext);
   const { section } = params;
-  const navItems = getFilteredIntegrationSettingsNavItems(isSandboxMode);
+  const navItems = integrationSettingsNavItems;
   const DEFAULT_SETTINGS_SECTION = navItems[0];
   const currentSection =
     navItems.find((item) => item.urlSection === section) ??
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
new file mode 100644
index 000000000..de7c79a13
--- /dev/null
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
@@ -0,0 +1,401 @@
+import React, { useState, useContext, useCallback } from "react";
+import { useQuery } from "react-query";
+
+import { IConfig } from "interfaces/config";
+import { NotificationContext } from "context/notification";
+import { AppContext } from "context/app";
+import configAPI from "services/entities/config";
+// @ts-ignore
+import { stringToClipboard } from "utilities/copy_text";
+
+// @ts-ignore
+import InputField from "components/forms/fields/InputField";
+import Button from "components/buttons/Button";
+import SectionHeader from "components/SectionHeader";
+import CustomLink from "components/CustomLink";
+import Spinner from "components/Spinner";
+import DataError from "components/DataError";
+import PremiumFeatureMessage from "components/PremiumFeatureMessage/PremiumFeatureMessage";
+import Icon from "components/Icon";
+
+const CREATING_SERVICE_ACCOUNT =
+  "https://www.fleetdm.com/learn-more-about/creating-service-accounts";
+const GOOGLE_WORKSPACE_DOMAINS =
+  "https://www.fleetdm.com/learn-more-about/google-workspace-domains";
+const DOMAIN_WIDE_DELEGATION =
+  "https://www.fleetdm.com/learn-more-about/domain-wide-delegation";
+const ENABLING_CALENDAR_API =
+  "fleetdm.com/learn-more-about/enabling-calendar-api";
+const OAUTH_SCOPES =
+  "https://www.googleapis.com/auth/calendar.events,https://www.googleapis.com/auth/calendar.settings.readonly";
+
+const API_KEY_JSON_PLACEHOLDER = `{
+  "type": "service_account",
+  "project_id": "fleet-in-your-calendar",
+  "private_key_id": "",
+  "private_key": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n",
+  "client_email": "fleet-calendar-events@fleet-in-your-calendar.iam.gserviceaccount.com",
+  "client_id": "",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/fleet-calendar-events%40fleet-in-your-calendar.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}`;
+
+interface IFormField {
+  name: string;
+  value: string | boolean | number;
+}
+
+interface ICalendarsFormErrors {
+  domain?: string | null;
+  apiKeyJson?: string | null;
+}
+
+interface ICalendarsFormData {
+  domain?: string;
+  apiKeyJson?: string;
+}
+
+const baseClass = "calendars-integration";
+
+const Calendars = (): JSX.Element => {
+  const { renderFlash } = useContext(NotificationContext);
+  const { isPremiumTier } = useContext(AppContext);
+
+  const [formData, setFormData] = useState({
+    domain: "",
+    apiKeyJson: "",
+  });
+  const [isUpdatingSettings, setIsUpdatingSettings] = useState(false);
+  const [formErrors, setFormErrors] = useState({});
+  const [copyMessage, setCopyMessage] = useState("");
+
+  const {
+    isLoading: isLoadingAppConfig,
+    refetch: refetchConfig,
+    error: errorAppConfig,
+  } = useQuery(["config"], () => configAPI.loadAll(), {
+    select: (data: IConfig) => data,
+    onSuccess: (data) => {
+      if (data.integrations.google_calendar) {
+        setFormData({
+          domain: data.integrations.google_calendar[0].domain,
+          // Formats string for better UI readability
+          apiKeyJson: JSON.stringify(
+            data.integrations.google_calendar[0].api_key_json,
+            null,
+            "\t"
+          ),
+        });
+      }
+    },
+  });
+
+  const { apiKeyJson, domain } = formData;
+
+  const validateForm = (curFormData: ICalendarsFormData) => {
+    const errors: ICalendarsFormErrors = {};
+
+    // Must set all keys or no keys at all
+    if (!curFormData.apiKeyJson && !!curFormData.domain) {
+      errors.apiKeyJson = "API key JSON must be present";
+    }
+    if (!curFormData.domain && !!curFormData.apiKeyJson) {
+      errors.apiKeyJson = "Domain must be present";
+    }
+    return errors;
+  };
+
+  const onInputChange = useCallback(
+    ({ name, value }: IFormField) => {
+      const newFormData = { ...formData, [name]: value };
+      setFormData(newFormData);
+      setFormErrors(validateForm(newFormData));
+    },
+    [formData]
+  );
+
+  const onFormSubmit = async (evt: React.MouseEvent) => {
+    setIsUpdatingSettings(true);
+
+    evt.preventDefault();
+
+    // Format for API
+    const formDataToSubmit =
+      formData.apiKeyJson === "" && formData.domain === ""
+        ? [] // Send empty array if no keys are set
+        : [
+            {
+              domain: formData.domain,
+              api_key_json:
+                (formData.apiKeyJson && JSON.parse(formData.apiKeyJson)) ||
+                null,
+            },
+          ];
+
+    // Update integrations.google_calendar only
+    const destination = {
+      google_calendar: formDataToSubmit,
+    };
+
+    try {
+      await configAPI.update({ integrations: destination });
+      renderFlash(
+        "success",
+        "Successfully saved calendar integration settings"
+      );
+      refetchConfig();
+    } catch (e) {
+      renderFlash("error", "Could not save calendar integration settings");
+    } finally {
+      setIsUpdatingSettings(false);
+    }
+  };
+
+  const renderOauthLabel = () => {
+    const onCopyOauthScopes = (evt: React.MouseEvent) => {
+      evt.preventDefault();
+
+      stringToClipboard(OAUTH_SCOPES)
+        .then(() => setCopyMessage(() => "Copied!"))
+        .catch(() => setCopyMessage(() => "Copy failed"));
+
+      // Clear message after 1 second
+      setTimeout(() => setCopyMessage(() => ""), 1000);
+
+      return false;
+    };
+
+    return (
+      
+        
+        {copyMessage && (
+          {copyMessage}
+        )}
+      
+    );
+  };
+
+  const renderForm = () => {
+    return (
+      <>
+        
+        

+          To create calendar events for end users with failing policies,
+          you'll need to configure a dedicated Google Workspace service
+          account.
+        

+        

+          

+            1. Go to the Service Accounts page in Google Cloud Platform.{" "}
+            
+          

+          

+            2. Create a new project for your service account.
+            
    
+              
  • 
+                Click Create project.
+              
    • 
+              
  • 
+                Enter "Fleet calendar events" as the project name.
+              
    • 
+              
  • 
+                For "Organization" and "Location", select
+                your calendar's organization.
+              
    • 
+            

+          

+
+          

+            3. Create the service account.
+            
    
+              
  • 
+                Click Create service account.
+              
    • 
+              
  • 
+                Set the service account name to "Fleet calendar
+                events".
+              
    • 
+              
  • 
+                Set the service account ID to "fleet-calendar-events".
+              
    • 
+              
  • 
+                Click Create and continue.
+              
    • 
+              
  • 
+                Click Done at the bottom of the form. (No need to
+                complete the optional steps.)
+              
    • 
+            

+          

+          

+            4. Create an API key.{" "}
+            
    
+              
  • 
+                Click the Actions menu for your new service account.
+              
    • 
+              
  • 
+                Select Manage keys.
+              
    • 
+              
  • 
+                Click Add key > Create new key.
+              
    • 
+              
  • Select the JSON key type.
    • 
+              
  • 
+                Click Create to create the key & download a JSON file.
+              
    • 
+              
  • 
+                Configure your service account integration in Fleet using the
+                form below:
+                
    
+                  
+                        Paste the full contents of the JSON file downloaded{" "}
+                        
    
+                        when creating your service account API key.
+                      
+                    }
+                    placeholder={API_KEY_JSON_PLACEHOLDER}
+                    ignore1password
+                    inputClassName={`${baseClass}__api-key-json`}
+                  />
+                  
+                        If the end user is signed into multiple Google accounts,
+                        this will be used to identify their work calendar.
+                      
+                    }
+                    placeholder="example.com"
+                    helpText={
+                      <>
+                        You can find your primary domain in Google Workspace{" "}
+                        
+                      
+                    }
+                  />
+                  
+                
+              
    • 
+            

+          

+          

+            5. Authorize the service account via domain-wide delegation.
+            
    
+              
  • 
+                In Google Workspace, go to{" "}
+                
+                  Security > Access and data control > API controls >
+                  Manage Domain Wide Delegation
+                
+                .{" "}
+                
+              
    • 
+              
  • 
+                Under API clients, click Add new.
+              
    • 
+              
  • 
+                Enter the client ID for the service account. You can find this
+                in your downloaded API key JSON file (
+                client_id
+                ), or under Advanced Settings when viewing the service
+                account.
+              
    • 
+              
  • 
+                For the OAuth scopes, paste the following value:
+                
+              
    • 
+              
  • 
+                Click Authorize.
+              
    • 
+            

+          

+          

+            6. Enable the Google Calendar API.
+            
    
+              
  • 
+                In the Google Cloud console API library, go to the Google
+                Calendar API.{" "}
+                
+              
    • 
+              
  • 
+                Make sure the "Fleet calendar events" project is
+                selected at the top of the page.
+              
    • 
+              
  • 
+                Click Enable.
+              
    • 
+            

+          

+        

+      
+    );
+  };
+
+  if (!isPremiumTier) return ;
+
+  if (isLoadingAppConfig) {
+    

+      
+    
;
+  }
+
+  if (errorAppConfig) {
+    return ;
+  }
+
+  return 
{renderForm()}
;
+};
+
+export default Calendars;
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss
new file mode 100644
index 000000000..7045a443e
--- /dev/null
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss
@@ -0,0 +1,57 @@
+.calendars-integration {
+  &__page-description {
+    font-size: $x-small;
+    color: $core-fleet-black;
+  }
+
+  ui {
+    margin-block-start: $pad-small;
+  }
+
+  li {
+    margin: $pad-small 0;
+  }
+
+  form {
+    margin-top: $pad-large;
+  }
+
+  &__configuration {
+    button {
+      align-self: flex-end;
+    }
+  }
+
+  &__api-key-json {
+    min-width: 100%; // resize vertically only
+    height: 294px;
+    font-size: $x-small;
+  }
+
+  #oauth-scopes {
+    font-family: "SourceCodePro", $monospace;
+    min-height: 80px;
+    padding: $pad-medium;
+    padding-right: $pad-xxlarge;
+    resize: none;
+  }
+
+  &__oauth-scopes-copy-icon-wrapper {
+    display: flex;
+    flex-direction: row-reverse;
+    align-items: center;
+    position: relative;
+    top: 36px;
+    right: 16px;
+    height: 0;
+    gap: 0.5rem;
+  }
+
+  &__copy-message {
+    @include copy-message;
+  }
+
+  &__code {
+    font-family: "SourceCodePro", $monospace;
+  }
+}
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts b/frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts
new file mode 100644
index 000000000..99dcd737c
--- /dev/null
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/index.ts
@@ -0,0 +1 @@
+export { default } from "./Calendars";
diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx
index 75f95f836..abe8aef4d 100644
--- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx
+++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/Integrations.tsx
@@ -8,7 +8,7 @@ import {
   IZendeskIntegration,
   IIntegration,
   IIntegrationTableData,
-  IIntegrations,
+  IGlobalIntegrations,
 } from "interfaces/integration";
 import { IApiError } from "interfaces/errors";
 
@@ -69,7 +69,7 @@ const Integrations = (): JSX.Element => {
     isLoading: isLoadingIntegrations,
     error: loadingIntegrationsError,
     refetch: refetchIntegrations,
-  } = useQuery(
+  } = useQuery(
     ["integrations"],
     () => configAPI.loadAll(),
     {
@@ -133,9 +133,15 @@ const Integrations = (): JSX.Element => {
       // Updates either integrations.jira or integrations.zendesk
       const destination = () => {
         if (integrationDestination === "jira") {
-          return { jira: integrationSubmitData, zendesk: zendeskIntegrations };
+          return {
+            jira: integrationSubmitData,
+            zendesk: zendeskIntegrations,
+          };
         }
-        return { zendesk: integrationSubmitData, jira: jiraIntegrations };
+        return {
+          zendesk: integrationSubmitData,
+          jira: jiraIntegrations,
+        };
       };
 
       setTestingConnection(true);
diff --git a/frontend/pages/admin/components/SideNav/SideNav.tsx b/frontend/pages/admin/components/SideNav/SideNav.tsx
index 1242cdcba..333f3f3f9 100644
--- a/frontend/pages/admin/components/SideNav/SideNav.tsx
+++ b/frontend/pages/admin/components/SideNav/SideNav.tsx
@@ -12,7 +12,6 @@ export interface ISideNavItem {
   urlSection: string;
   path: string;
   Card: (props: T) => JSX.Element;
-  exclude?: boolean;
 }
 
 interface ISideNavProps {
diff --git a/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx
index 907f4edb4..54622a692 100644
--- a/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx
+++ b/frontend/pages/policies/ManagePoliciesPage/components/OtherWorkflowsModal/OtherWorkflowsModal.tsx
@@ -3,7 +3,12 @@ import { Link } from "react-router";
 import { isEmpty, noop, omit } from "lodash";
 
 import { IAutomationsConfig, IWebhookSettings } from "interfaces/config";
-import { IIntegration, IIntegrations } from "interfaces/integration";
+import {
+  IGlobalIntegrations,
+  IIntegration,
+  IIntegrations,
+  ITeamIntegrations,
+} from "interfaces/integration";
 import { IPolicy } from "interfaces/policy";
 import { ITeamAutomationsConfig } from "interfaces/team";
 import PATHS from "router/paths";
@@ -26,13 +31,13 @@ import ExamplePayload from "../ExamplePayload";
 
 interface IOtherWorkflowsModalProps {
   automationsConfig: IAutomationsConfig | ITeamAutomationsConfig;
-  availableIntegrations: IIntegrations;
+  availableIntegrations: IGlobalIntegrations | ITeamIntegrations;
   availablePolicies: IPolicy[];
   isUpdatingAutomations: boolean;
   onExit: () => void;
   handleSubmit: (formData: {
     webhook_settings: Pick;
-    integrations: IIntegrations;
+    integrations: IGlobalIntegrations | ITeamIntegrations;
   }) => void;
 }
 
@@ -256,6 +261,7 @@ const OtherWorkflowsModal = ({
       integrations: {
         jira: newJira,
         zendesk: newZendesk,
+        google_calendar: null, // When null, the backend does not update google_calendar
       },
     });
 
diff --git a/frontend/router/paths.ts b/frontend/router/paths.ts
index 114e42dd8..4a1752c65 100644
--- a/frontend/router/paths.ts
+++ b/frontend/router/paths.ts
@@ -32,6 +32,7 @@ export default {
   ADMIN_INTEGRATIONS_MDM_WINDOWS: `${URL_PREFIX}/settings/integrations/mdm/windows`,
   ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT: `${URL_PREFIX}/settings/integrations/automatic-enrollment`,
   ADMIN_INTEGRATIONS_AUTOMATIC_ENROLLMENT_WINDOWS: `${URL_PREFIX}/settings/integrations/automatic-enrollment/windows`,
+  ADMIN_INTEGRATIONS_CALENDARS: `${URL_PREFIX}/settings/integrations/calendars`,
   ADMIN_TEAMS: `${URL_PREFIX}/settings/teams`,
   ADMIN_ORGANIZATION: `${URL_PREFIX}/settings/organization`,
   ADMIN_ORGANIZATION_INFO: `${URL_PREFIX}/settings/organization/info`,
diff --git a/frontend/styles/var/mixins.scss b/frontend/styles/var/mixins.scss
index a6ef009b6..3751ffadf 100644
--- a/frontend/styles/var/mixins.scss
+++ b/frontend/styles/var/mixins.scss
@@ -182,6 +182,15 @@ $max-width: 2560px;
   }
 }
 
+@mixin copy-message {
+  font-weight: $regular;
+  vertical-align: top;
+  background-color: $ui-light-grey;
+  border: solid 1px #e2e4ea;
+  border-radius: 10px;
+  padding: 2px 6px;
+}
+
 @mixin color-contrasted-sections {
   background-color: $ui-off-white;
   .section {

From 16f122f02a697b3d974395c97d3fbf6eb68aafdc Mon Sep 17 00:00:00 2001
From: Victor Lyuboslavsky 
Date: Fri, 22 Mar 2024 09:19:55 -0500
Subject: [PATCH 17/36] Adding calendar test server and other fixes. (#17751)

- Added a calendar server that can be used for load testing at
/tools/calendar
- Fixed minor calendar bugs

# Checklist for submitter
- [ ] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [x] Added/updated tests
- [x] Manual QA for all new/changed functionality
---
 ee/server/calendar/google_calendar.go         | 109 ++++--
 .../google_calendar_integration_test.go       | 132 +++++++
 ee/server/calendar/google_calendar_load.go    | 234 ++++++++++++
 ee/server/calendar/google_calendar_mock.go    |   8 +-
 ee/server/calendar/google_calendar_test.go    |  30 ++
 .../load_test/calendar_http_handler.go        | 343 ++++++++++++++++++
 tools/calendar/README.md                      |  26 ++
 tools/calendar/calendar.go                    |  39 ++
 8 files changed, 882 insertions(+), 39 deletions(-)
 create mode 100644 ee/server/calendar/google_calendar_integration_test.go
 create mode 100644 ee/server/calendar/google_calendar_load.go
 create mode 100644 ee/server/calendar/load_test/calendar_http_handler.go
 create mode 100644 tools/calendar/README.md
 create mode 100644 tools/calendar/calendar.go

diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go
index 26d1ba1e6..42f8b7b0d 100644
--- a/ee/server/calendar/google_calendar.go
+++ b/ee/server/calendar/google_calendar.go
@@ -6,6 +6,8 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"os"
+	"regexp"
 	"time"
 
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
@@ -19,18 +21,30 @@ import (
 	"google.golang.org/api/option"
 )
 
+// The calendar package has the following features for testing:
+// 1. High level UserCalendar interface and Low level GoogleCalendarAPI interface can have a custom implementations.
+// 2. Setting "client_email" to "calendar-mock@example.com" in the API key will use a mock in-memory implementation GoogleCalendarMockAPI of GoogleCalendarAPI.
+// 3. Setting FLEET_GOOGLE_CALENDAR_PLUS_ADDRESSING environment variable to "1" will strip the "plus addressing" from the user email, effectively allowing a single user
+//    to create multiple events in the same calendar. This is useful for load testing. For example: john+test@example.com becomes john@example.com
+
 const (
 	eventTitle  = "💻🚫Downtime"
 	startHour   = 9
 	endHour     = 17
 	eventLength = 30 * time.Minute
 	calendarID  = "primary"
+	mockEmail   = "calendar-mock@example.com"
+	loadEmail   = "calendar-load@example.com"
 )
 
-var calendarScopes = []string{
-	"https://www.googleapis.com/auth/calendar.events",
-	"https://www.googleapis.com/auth/calendar.settings.readonly",
-}
+var (
+	calendarScopes = []string{
+		"https://www.googleapis.com/auth/calendar.events",
+		"https://www.googleapis.com/auth/calendar.settings.readonly",
+	}
+	plusAddressing      = os.Getenv("FLEET_GOOGLE_CALENDAR_PLUS_ADDRESSING") == "1"
+	plusAddressingRegex = regexp.MustCompile(`\+.*@`)
+)
 
 type GoogleCalendarConfig struct {
 	Context           context.Context
@@ -43,19 +57,22 @@ type GoogleCalendarConfig struct {
 // GoogleCalendar is an implementation of the UserCalendar interface that uses the
 // Google Calendar API to manage events.
 type GoogleCalendar struct {
-	config           *GoogleCalendarConfig
-	currentUserEmail string
-	timezoneOffset   *int
+	config            *GoogleCalendarConfig
+	currentUserEmail  string
+	adjustedUserEmail string
+	location          *time.Location
 }
 
 func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar {
-	if config.API == nil {
-		if config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == "calendar-mock@example.com" {
-			// Assumes that only 1 Fleet server accesses the calendar, since all mock events are held in memory
-			config.API = &GoogleCalendarMockAPI{}
-		} else {
-			config.API = &GoogleCalendarLowLevelAPI{}
-		}
+	switch {
+	case config.API != nil:
+		// Use the provided API.
+	case config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == loadEmail:
+		config.API = &GoogleCalendarLoadAPI{Logger: config.Logger}
+	case config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == mockEmail:
+		config.API = &GoogleCalendarMockAPI{config.Logger}
+	default:
+		config.API = &GoogleCalendarLowLevelAPI{}
 	}
 	return &GoogleCalendar{
 		config: config,
@@ -101,6 +118,13 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) Configure(
 	return nil
 }
 
+func adjustEmail(email string) string {
+	if plusAddressing {
+		return plusAddressingRegex.ReplaceAllString(email, "@")
+	}
+	return email
+}
+
 func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) {
 	return lowLevelAPI.service.Settings.Get(name).Do()
 }
@@ -130,14 +154,18 @@ func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error {
 }
 
 func (c *GoogleCalendar) Configure(userEmail string) error {
+	adjustedUserEmail := adjustEmail(userEmail)
 	err := c.config.API.Configure(
 		c.config.Context, c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail],
-		c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarPrivateKey], userEmail,
+		c.config.IntegrationConfig.ApiKey[fleet.GoogleCalendarPrivateKey], adjustedUserEmail,
 	)
 	if err != nil {
 		return ctxerr.Wrap(c.config.Context, err, "creating Google calendar service")
 	}
 	c.currentUserEmail = userEmail
+	c.adjustedUserEmail = adjustedUserEmail
+	// Clear the timezone offset so that it will be recalculated
+	c.location = nil
 	return nil
 }
 
@@ -162,7 +190,7 @@ func (c *GoogleCalendar) GetAndUpdateEvent(event *fleet.CalendarEvent, genBodyFn
 		return nil, false, ctxerr.Wrap(c.config.Context, err, "retrieving Google calendar event")
 	}
 	if !deleted && gEvent.Status != "cancelled" {
-		if details.ETag == gEvent.Etag {
+		if details.ETag != "" && details.ETag == gEvent.Etag {
 			// Event was not modified
 			return event, false, nil
 		}
@@ -246,20 +274,20 @@ func calculateNewEventDate(oldStartDate time.Time) time.Time {
 }
 
 func (c *GoogleCalendar) parseDateTime(eventDateTime *calendar.EventDateTime) (*time.Time, error) {
-	var endTime time.Time
+	var t time.Time
 	var err error
 	if eventDateTime.TimeZone != "" {
 		loc := getLocation(eventDateTime.TimeZone, c.config)
-		endTime, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc)
+		t, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc)
 	} else {
-		endTime, err = time.Parse(time.RFC3339, eventDateTime.DateTime)
+		t, err = time.Parse(time.RFC3339, eventDateTime.DateTime)
 	}
 	if err != nil {
 		return nil, ctxerr.Wrap(
 			c.config.Context, err, fmt.Sprintf("parsing Google calendar event time: %s", eventDateTime.DateTime),
 		)
 	}
-	return &endTime, nil
+	return &t, nil
 }
 
 func isNotFound(err error) bool {
@@ -271,6 +299,15 @@ func isNotFound(err error) bool {
 	return ok && ae.Code == http.StatusNotFound
 }
 
+func isAlreadyDeleted(err error) bool {
+	if err == nil {
+		return false
+	}
+	var ae *googleapi.Error
+	ok := errors.As(err, &ae)
+	return ok && ae.Code == http.StatusGone
+}
+
 func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) {
 	var details eventDetails
 	err := json.Unmarshal(event.Data, &details)
@@ -293,18 +330,18 @@ func (c *GoogleCalendar) CreateEvent(dayOfEvent time.Time, genBodyFn func(confli
 func (c *GoogleCalendar) createEvent(
 	dayOfEvent time.Time, genBodyFn func(conflict bool) string, timeNow func() time.Time,
 ) (*fleet.CalendarEvent, error) {
-	if c.timezoneOffset == nil {
-		err := getTimezone(c)
+	var err error
+	if c.location == nil {
+		c.location, err = getTimezone(c)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	location := time.FixedZone("", *c.timezoneOffset)
-	dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, location)
-	dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, location)
+	dayStart := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), startHour, 0, 0, 0, c.location)
+	dayEnd := time.Date(dayOfEvent.Year(), dayOfEvent.Month(), dayOfEvent.Day(), endHour, 0, 0, 0, c.location)
 
-	now := timeNow().In(location)
+	now := timeNow().In(c.location)
 	if dayEnd.Before(now) {
 		// The workday has already ended.
 		return nil, ctxerr.Wrap(c.config.Context, fleet.DayEndedError{Msg: "cannot schedule an event for a day that has already ended"})
@@ -342,7 +379,7 @@ func (c *GoogleCalendar) createEvent(
 		// Ignore events that the user has declined
 		var declined bool
 		for _, attendee := range gEvent.Attendees {
-			if attendee.Email == c.currentUserEmail {
+			if attendee.Email == c.adjustedUserEmail {
 				// The user has declined the event, so this time is open for scheduling
 				if attendee.ResponseStatus == "declined" {
 					declined = true
@@ -396,7 +433,9 @@ func (c *GoogleCalendar) createEvent(
 	if err != nil {
 		return nil, err
 	}
-	level.Debug(c.config.Logger).Log("msg", "created Google calendar event", "user", c.currentUserEmail, "startTime", eventStart)
+	level.Debug(c.config.Logger).Log(
+		"msg", "created Google calendar event", "user", c.adjustedUserEmail, "startTime", eventStart, "timezone", c.location.String(),
+	)
 
 	return fleetEvent, nil
 }
@@ -419,17 +458,14 @@ func adjustEventTimes(endTime time.Time, dayEnd time.Time) (eventStart time.Time
 	return eventStart, eventEnd, isLastSlot, conflict
 }
 
-func getTimezone(gCal *GoogleCalendar) error {
+func getTimezone(gCal *GoogleCalendar) (*time.Location, error) {
 	config := gCal.config
 	setting, err := config.API.GetSetting("timezone")
 	if err != nil {
-		return ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone")
+		return nil, ctxerr.Wrap(config.Context, err, "retrieving Google calendar timezone")
 	}
 
-	loc := getLocation(setting.Value, config)
-	_, timezoneOffset := time.Now().In(loc).Zone()
-	gCal.timezoneOffset = &timezoneOffset
-	return nil
+	return getLocation(setting.Value, config), nil
 }
 
 func getLocation(name string, config *GoogleCalendarConfig) *time.Location {
@@ -467,7 +503,10 @@ func (c *GoogleCalendar) DeleteEvent(event *fleet.CalendarEvent) error {
 		return err
 	}
 	err = c.config.API.DeleteEvent(details.ID)
-	if err != nil {
+	switch {
+	case isAlreadyDeleted(err):
+		return nil
+	case err != nil:
 		return ctxerr.Wrap(c.config.Context, err, "deleting Google calendar event")
 	}
 	return nil
diff --git a/ee/server/calendar/google_calendar_integration_test.go b/ee/server/calendar/google_calendar_integration_test.go
new file mode 100644
index 000000000..7f42b23b2
--- /dev/null
+++ b/ee/server/calendar/google_calendar_integration_test.go
@@ -0,0 +1,132 @@
+package calendar
+
+import (
+	"context"
+	"github.com/fleetdm/fleet/v4/ee/server/calendar/load_test"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	kitlog "github.com/go-kit/log"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+)
+
+type googleCalendarIntegrationTestSuite struct {
+	suite.Suite
+	server *httptest.Server
+	dbFile *os.File
+}
+
+func (s *googleCalendarIntegrationTestSuite) SetupSuite() {
+	dbFile, err := os.CreateTemp("", "calendar.db")
+	s.Require().NoError(err)
+	handler, err := calendartest.Configure(dbFile.Name())
+	s.Require().NoError(err)
+	server := httptest.NewUnstartedServer(handler)
+	server.Listener.Addr()
+	server.Start()
+	s.server = server
+}
+
+func (s *googleCalendarIntegrationTestSuite) TearDownSuite() {
+	if s.dbFile != nil {
+		s.dbFile.Close()
+		_ = os.Remove(s.dbFile.Name())
+	}
+	if s.server != nil {
+		s.server.Close()
+	}
+	calendartest.Close()
+}
+
+// TestGoogleCalendarIntegration tests should be able to be run in parallel, but this is not natively supported by suites: https://github.com/stretchr/testify/issues/187
+// There are workarounds that can be explored.
+func TestGoogleCalendarIntegration(t *testing.T) {
+	testingSuite := new(googleCalendarIntegrationTestSuite)
+	suite.Run(t, testingSuite)
+}
+
+func (s *googleCalendarIntegrationTestSuite) TestCreateGetDeleteEvent() {
+	t := s.T()
+	userEmail := "user1@example.com"
+	config := &GoogleCalendarConfig{
+		Context: context.Background(),
+		IntegrationConfig: &fleet.GoogleCalendarIntegration{
+			Domain: "example.com",
+			ApiKey: map[string]string{
+				"client_email": loadEmail,
+				"private_key":  s.server.URL,
+			},
+		},
+		Logger: kitlog.NewLogfmtLogger(kitlog.NewSyncWriter(os.Stdout)),
+	}
+	gCal := NewGoogleCalendar(config)
+	err := gCal.Configure(userEmail)
+	require.NoError(t, err)
+	genBodyFn := func(bool) string {
+		return "Test event"
+	}
+	eventDate := time.Now().Add(48 * time.Hour)
+	event, err := gCal.CreateEvent(eventDate, genBodyFn)
+	require.NoError(t, err)
+	assert.Equal(t, startHour, event.StartTime.Hour())
+	assert.Equal(t, 0, event.StartTime.Minute())
+
+	eventRsp, updated, err := gCal.GetAndUpdateEvent(event, genBodyFn)
+	require.NoError(t, err)
+	assert.False(t, updated)
+	assert.Equal(t, event, eventRsp)
+
+	err = gCal.DeleteEvent(event)
+	assert.NoError(t, err)
+	// delete again
+	err = gCal.DeleteEvent(event)
+	assert.NoError(t, err)
+
+	// Try to get deleted event
+	eventRsp, updated, err = gCal.GetAndUpdateEvent(event, genBodyFn)
+	require.NoError(t, err)
+	assert.True(t, updated)
+	assert.NotEqual(t, event.StartTime.UTC().Truncate(24*time.Hour), eventRsp.StartTime.UTC().Truncate(24*time.Hour))
+}
+
+func (s *googleCalendarIntegrationTestSuite) TestFillUpCalendar() {
+	t := s.T()
+	userEmail := "user2@example.com"
+	config := &GoogleCalendarConfig{
+		Context: context.Background(),
+		IntegrationConfig: &fleet.GoogleCalendarIntegration{
+			Domain: "example.com",
+			ApiKey: map[string]string{
+				"client_email": loadEmail,
+				"private_key":  s.server.URL,
+			},
+		},
+		Logger: kitlog.NewLogfmtLogger(kitlog.NewSyncWriter(os.Stdout)),
+	}
+	gCal := NewGoogleCalendar(config)
+	err := gCal.Configure(userEmail)
+	require.NoError(t, err)
+	genBodyFn := func(bool) string {
+		return "Test event"
+	}
+	eventDate := time.Now().Add(48 * time.Hour)
+	event, err := gCal.CreateEvent(eventDate, genBodyFn)
+	require.NoError(t, err)
+	assert.Equal(t, startHour, event.StartTime.Hour())
+	assert.Equal(t, 0, event.StartTime.Minute())
+
+	currentEventTime := event.StartTime
+	for i := 0; i < 20; i++ {
+		if !(currentEventTime.Hour() == endHour-1 && currentEventTime.Minute() == 30) {
+			currentEventTime = currentEventTime.Add(30 * time.Minute)
+		}
+		event, err = gCal.CreateEvent(eventDate, genBodyFn)
+		require.NoError(t, err)
+		assert.Equal(t, currentEventTime.UTC(), event.StartTime.UTC())
+	}
+
+}
diff --git a/ee/server/calendar/google_calendar_load.go b/ee/server/calendar/google_calendar_load.go
new file mode 100644
index 000000000..8446af20c
--- /dev/null
+++ b/ee/server/calendar/google_calendar_load.go
@@ -0,0 +1,234 @@
+package calendar
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
+	kitlog "github.com/go-kit/log"
+	"google.golang.org/api/calendar/v3"
+	"google.golang.org/api/googleapi"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+)
+
+// GoogleCalendarLoadAPI is used for load testing.
+type GoogleCalendarLoadAPI struct {
+	Logger            kitlog.Logger
+	baseUrl           string
+	userToImpersonate string
+	ctx               context.Context
+	client            *http.Client
+}
+
+// Configure creates a new Google Calendar service using the provided credentials.
+func (lowLevelAPI *GoogleCalendarLoadAPI) Configure(ctx context.Context, _ string, privateKey string, userToImpersonate string) error {
+	if lowLevelAPI.Logger == nil {
+		lowLevelAPI.Logger = kitlog.With(kitlog.NewLogfmtLogger(os.Stderr), "mock", "GoogleCalendarLoadAPI", "user", userToImpersonate)
+	}
+	lowLevelAPI.baseUrl = privateKey
+	lowLevelAPI.userToImpersonate = userToImpersonate
+	lowLevelAPI.ctx = ctx
+	if lowLevelAPI.client == nil {
+		lowLevelAPI.client = fleethttp.NewClient()
+	}
+	return nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) GetSetting(name string) (*calendar.Setting, error) {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/settings")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("name", name)
+	query.Set("email", lowLevelAPI.userToImpersonate)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "GET", reqUrl.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var setting calendar.Setting
+	body, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &setting)
+	if err != nil {
+		return nil, err
+	}
+	return &setting, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) {
+	body, err := json.Marshal(event)
+	if err != nil {
+		return nil, err
+	}
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events/add")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("email", lowLevelAPI.userToImpersonate)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "POST", reqUrl.String(), bytes.NewBuffer(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode != http.StatusCreated {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var rspEvent calendar.Event
+	body, err = io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &rspEvent)
+	if err != nil {
+		return nil, err
+	}
+	return &rspEvent, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) GetEvent(id, _ string) (*calendar.Event, error) {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("id", id)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "GET", reqUrl.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode == http.StatusNotFound {
+		return nil, &googleapi.Error{Code: http.StatusNotFound}
+	}
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var rspEvent calendar.Event
+	body, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &rspEvent)
+	if err != nil {
+		return nil, err
+	}
+	return &rspEvent, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) ListEvents(timeMin string, timeMax string) (*calendar.Events, error) {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events/list")
+	if err != nil {
+		return nil, err
+	}
+	query := reqUrl.Query()
+	query.Set("timemin", timeMin)
+	query.Set("timemax", timeMax)
+	query.Set("email", lowLevelAPI.userToImpersonate)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "GET", reqUrl.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return nil, fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	var events calendar.Events
+	body, err := io.ReadAll(rsp.Body)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &events)
+	if err != nil {
+		return nil, err
+	}
+	return &events, nil
+}
+
+func (lowLevelAPI *GoogleCalendarLoadAPI) DeleteEvent(id string) error {
+	reqUrl, err := url.Parse(lowLevelAPI.baseUrl + "/events/delete")
+	if err != nil {
+		return err
+	}
+	query := reqUrl.Query()
+	query.Set("id", id)
+	reqUrl.RawQuery = query.Encode()
+	req, err := http.NewRequestWithContext(lowLevelAPI.ctx, "DELETE", reqUrl.String(), nil)
+	if err != nil {
+		return err
+	}
+	rsp, err := lowLevelAPI.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = rsp.Body.Close()
+	}()
+	if rsp.StatusCode == http.StatusGone {
+		return &googleapi.Error{Code: http.StatusGone}
+	}
+	if rsp.StatusCode != http.StatusOK {
+		var data []byte
+		if rsp.Body != nil {
+			data, _ = io.ReadAll(rsp.Body)
+		}
+		return fmt.Errorf("unexpected status code: %d with body: %s", rsp.StatusCode, string(data))
+	}
+	return nil
+}
diff --git a/ee/server/calendar/google_calendar_mock.go b/ee/server/calendar/google_calendar_mock.go
index a6d7d6040..255f8d87c 100644
--- a/ee/server/calendar/google_calendar_mock.go
+++ b/ee/server/calendar/google_calendar_mock.go
@@ -17,7 +17,7 @@ type GoogleCalendarMockAPI struct {
 	logger kitlog.Logger
 }
 
-var events = make(map[string]*calendar.Event)
+var mockEvents = make(map[string]*calendar.Event)
 var mu sync.Mutex
 var id uint64
 
@@ -49,7 +49,7 @@ func (lowLevelAPI *GoogleCalendarMockAPI) CreateEvent(event *calendar.Event) (*c
 	id += 1
 	event.Id = strconv.FormatUint(id, 10)
 	lowLevelAPI.logger.Log("msg", "CreateEvent", "id", event.Id, "start", event.Start.DateTime)
-	events[event.Id] = event
+	mockEvents[event.Id] = event
 	return event, nil
 }
 
@@ -57,7 +57,7 @@ func (lowLevelAPI *GoogleCalendarMockAPI) GetEvent(id, _ string) (*calendar.Even
 	time.Sleep(latency)
 	mu.Lock()
 	defer mu.Unlock()
-	event, ok := events[id]
+	event, ok := mockEvents[id]
 	if !ok {
 		return nil, &googleapi.Error{Code: http.StatusNotFound}
 	}
@@ -76,6 +76,6 @@ func (lowLevelAPI *GoogleCalendarMockAPI) DeleteEvent(id string) error {
 	mu.Lock()
 	defer mu.Unlock()
 	lowLevelAPI.logger.Log("msg", "DeleteEvent", "id", id)
-	delete(events, id)
+	delete(mockEvents, id)
 	return nil
 }
diff --git a/ee/server/calendar/google_calendar_test.go b/ee/server/calendar/google_calendar_test.go
index ad5e1c89c..02d024792 100644
--- a/ee/server/calendar/google_calendar_test.go
+++ b/ee/server/calendar/google_calendar_test.go
@@ -84,6 +84,29 @@ func TestGoogleCalendar_Configure(t *testing.T) {
 	assert.ErrorIs(t, err, assert.AnError)
 }
 
+func TestGoogleCalendar_ConfigurePlusAddressing(t *testing.T) {
+	// Do not run this test in t.Parallel(), since it involves modifying a global variable
+	plusAddressing = true
+	t.Cleanup(
+		func() {
+			plusAddressing = false
+		},
+	)
+	email := "user+my_test+email@example.com"
+	mockAPI := &MockGoogleCalendarLowLevelAPI{}
+	mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error {
+		assert.Equal(t, baseCtx, ctx)
+		assert.Equal(t, baseServiceEmail, serviceAccountEmail)
+		assert.Equal(t, basePrivateKey, privateKey)
+		assert.Equal(t, "user@example.com", userToImpersonateEmail)
+		return nil
+	}
+
+	var cal fleet.UserCalendar = NewGoogleCalendar(makeConfig(mockAPI))
+	err := cal.Configure(email)
+	assert.NoError(t, err)
+}
+
 func makeConfig(mockAPI *MockGoogleCalendarLowLevelAPI) *GoogleCalendarConfig {
 	if mockAPI != nil && mockAPI.ConfigureFunc == nil {
 		mockAPI.ConfigureFunc = func(ctx context.Context, serviceAccountEmail, privateKey, userToImpersonateEmail string) error {
@@ -125,6 +148,13 @@ func TestGoogleCalendar_DeleteEvent(t *testing.T) {
 	}
 	err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)})
 	assert.ErrorIs(t, err, assert.AnError)
+
+	// Event already deleted
+	mockAPI.DeleteEventFunc = func(id string) error {
+		return &googleapi.Error{Code: http.StatusGone}
+	}
+	err = cal.DeleteEvent(&fleet.CalendarEvent{Data: []byte(`{"ID":"event-id"}`)})
+	assert.NoError(t, err)
 }
 
 func TestGoogleCalendar_unmarshalDetails(t *testing.T) {
diff --git a/ee/server/calendar/load_test/calendar_http_handler.go b/ee/server/calendar/load_test/calendar_http_handler.go
new file mode 100644
index 000000000..e69c94552
--- /dev/null
+++ b/ee/server/calendar/load_test/calendar_http_handler.go
@@ -0,0 +1,343 @@
+// Package calendartest is not imported in production code, so it will not be compiled for Fleet server.
+package calendartest
+
+import (
+	"context"
+	"crypto/md5" //nolint:gosec // (only used in testing)
+	"database/sql"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	_ "github.com/mattn/go-sqlite3"
+	"google.golang.org/api/calendar/v3"
+	"hash/fnv"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+// This calendar does not support all-day events.
+
+var db *sql.DB
+var timezones = []string{
+	"America/Chicago",
+	"America/New_York",
+	"America/Los_Angeles",
+	"America/Anchorage",
+	"Pacific/Honolulu",
+	"America/Argentina/Buenos_Aires",
+	"Asia/Kolkata",
+	"Europe/London",
+	"Europe/Paris",
+	"Australia/Sydney",
+}
+
+func Configure(dbPath string) (http.Handler, error) {
+	var err error
+	db, err = sql.Open("sqlite3", dbPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	logger := log.New(os.Stdout, "", log.LstdFlags)
+	logger.Println("Server is starting...")
+
+	// Initialize the database schema if needed
+	err = initializeSchema()
+	if err != nil {
+		return nil, err
+	}
+
+	router := http.NewServeMux()
+	router.HandleFunc("/settings", getSetting)
+	router.HandleFunc("/events", getEvent)
+	router.HandleFunc("/events/list", getEvents)
+	router.HandleFunc("/events/add", addEvent)
+	router.HandleFunc("/events/delete", deleteEvent)
+	return logging(logger)(router), nil
+}
+
+func logging(logger *log.Logger) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(
+			func(w http.ResponseWriter, r *http.Request) {
+				defer func() {
+					logger.Println(r.Method, r.URL.String(), r.RemoteAddr)
+				}()
+				next.ServeHTTP(w, r)
+			},
+		)
+	}
+}
+
+func Close() {
+	_ = db.Close()
+}
+
+func getSetting(w http.ResponseWriter, r *http.Request) {
+	name := r.URL.Query().Get("name")
+	if name == "" {
+		http.Error(w, "missing name", http.StatusBadRequest)
+		return
+	}
+	if name != "timezone" {
+		http.Error(w, "unsupported setting", http.StatusNotFound)
+		return
+	}
+	email := r.URL.Query().Get("email")
+	if email == "" {
+		http.Error(w, "missing email", http.StatusBadRequest)
+		return
+	}
+	timezone := getTimezone(email)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	setting := calendar.Setting{Value: timezone}
+	err := json.NewEncoder(w).Encode(setting)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// The timezone is determined by the user's email address
+func getTimezone(email string) string {
+	index := hash(email) % uint32(len(timezones))
+	timezone := timezones[index]
+	return timezone
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(s))
+	return h.Sum32()
+}
+
+// getEvent handles GET /events?id=123
+func getEvent(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("id")
+	if id == "" {
+		http.Error(w, "missing id", http.StatusBadRequest)
+		return
+	}
+	sqlStmt := "SELECT email, start, end, summary, description, status FROM events WHERE id = ?"
+	var start, end int64
+	var email, summary, description, status string
+	err := db.QueryRow(sqlStmt, id).Scan(&email, &start, &end, &summary, &description, &status)
+	if errors.Is(err, sql.ErrNoRows) {
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	timezone := getTimezone(email)
+	loc, err := time.LoadLocation(timezone)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	calEvent := calendar.Event{}
+	calEvent.Id = id
+	calEvent.Start = &calendar.EventDateTime{DateTime: time.Unix(start, 0).In(loc).Format(time.RFC3339)}
+	calEvent.End = &calendar.EventDateTime{DateTime: time.Unix(end, 0).In(loc).Format(time.RFC3339)}
+	calEvent.Summary = summary
+	calEvent.Description = description
+	calEvent.Status = status
+	calEvent.Etag = computeETag(start, end, summary, description, status)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(w).Encode(calEvent)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func getEvents(w http.ResponseWriter, r *http.Request) {
+	email := r.URL.Query().Get("email")
+	if email == "" {
+		http.Error(w, "missing email", http.StatusBadRequest)
+		return
+	}
+	timeMin := r.URL.Query().Get("timemin")
+	if email == "" {
+		http.Error(w, "missing timemin", http.StatusBadRequest)
+		return
+	}
+	timeMax := r.URL.Query().Get("timemax")
+	if email == "" {
+		http.Error(w, "missing timemax", http.StatusBadRequest)
+		return
+	}
+	minTime, err := parseDateTime(r.Context(), &calendar.EventDateTime{DateTime: timeMin})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	maxTime, err := parseDateTime(r.Context(), &calendar.EventDateTime{DateTime: timeMax})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	sqlStmt := "SELECT id, start, end, summary, description, status FROM events WHERE email = ? AND end > ? AND start < ?"
+	rows, err := db.Query(sqlStmt, email, minTime.Unix(), maxTime.Unix())
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer rows.Close()
+	timezone := getTimezone(email)
+	loc, err := time.LoadLocation(timezone)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	events := calendar.Events{}
+	events.Items = make([]*calendar.Event, 0)
+	for rows.Next() {
+		var id, start, end int64
+		var summary, description, status string
+		err = rows.Scan(&id, &start, &end, &summary, &description, &status)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		calEvent := calendar.Event{}
+		calEvent.Id = fmt.Sprintf("%d", id)
+		calEvent.Start = &calendar.EventDateTime{DateTime: time.Unix(start, 0).In(loc).Format(time.RFC3339)}
+		calEvent.End = &calendar.EventDateTime{DateTime: time.Unix(end, 0).In(loc).Format(time.RFC3339)}
+		calEvent.Summary = summary
+		calEvent.Description = description
+		calEvent.Status = status
+		calEvent.Etag = computeETag(start, end, summary, description, status)
+		events.Items = append(events.Items, &calEvent)
+	}
+	if err = rows.Err(); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(w).Encode(events)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// addEvent handles POST /events/add?email=user@example.com
+func addEvent(w http.ResponseWriter, r *http.Request) {
+	var event calendar.Event
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	err = json.Unmarshal(body, &event)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	email := r.URL.Query().Get("email")
+	if email == "" {
+		http.Error(w, "missing email", http.StatusBadRequest)
+		return
+	}
+	start, err := parseDateTime(r.Context(), event.Start)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	end, err := parseDateTime(r.Context(), event.End)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	status := "confirmed"
+	sqlStmt := `INSERT INTO events (email, start, end, summary, description, status) VALUES (?, ?, ?, ?, ?, ?)`
+	result, err := db.Exec(sqlStmt, email, start.Unix(), end.Unix(), event.Summary, event.Description, status)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	id, err := result.LastInsertId()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	event.Id = fmt.Sprintf("%d", id)
+	event.Etag = computeETag(start.Unix(), end.Unix(), event.Summary, event.Description, status)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusCreated)
+	err = json.NewEncoder(w).Encode(event)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+func computeETag(args ...any) string {
+	h := md5.New() //nolint:gosec // (only used for tests)
+	_, _ = fmt.Fprint(h, args...)
+	checksum := h.Sum(nil)
+	return hex.EncodeToString(checksum)
+}
+
+// deleteEvent handles DELETE /events/delete?id=123
+func deleteEvent(w http.ResponseWriter, r *http.Request) {
+	id := r.URL.Query().Get("id")
+	if id == "" {
+		http.Error(w, "missing id", http.StatusBadRequest)
+		return
+	}
+	sqlStmt := "DELETE FROM events WHERE id = ?"
+	_, err := db.Exec(sqlStmt, id)
+	if errors.Is(err, sql.ErrNoRows) {
+		http.Error(w, "not found", http.StatusGone)
+		return
+	}
+}
+
+func initializeSchema() error {
+	createTableSQL := `CREATE TABLE IF NOT EXISTS events (
+		"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+		"email" TEXT NOT NULL,
+		"start" INTEGER NOT NULL,
+		"end" INTEGER NOT NULL,
+		"summary" TEXT NOT NULL,
+		"description" TEXT NOT NULL,
+		"status" TEXT NOT NULL
+	);`
+	_, err := db.Exec(createTableSQL)
+	if err != nil {
+		return fmt.Errorf("failed to create table: %w", err)
+	}
+	return nil
+}
+
+func parseDateTime(ctx context.Context, eventDateTime *calendar.EventDateTime) (*time.Time, error) {
+	var t time.Time
+	var err error
+	if eventDateTime.TimeZone != "" {
+		var loc *time.Location
+		loc, err = time.LoadLocation(eventDateTime.TimeZone)
+		if err == nil {
+			t, err = time.ParseInLocation(time.RFC3339, eventDateTime.DateTime, loc)
+		}
+	} else {
+		t, err = time.Parse(time.RFC3339, eventDateTime.DateTime)
+	}
+	if err != nil {
+		return nil, ctxerr.Wrap(
+			ctx, err, fmt.Sprintf("parsing calendar event time: %s", eventDateTime.DateTime),
+		)
+	}
+	return &t, nil
+}
diff --git a/tools/calendar/README.md b/tools/calendar/README.md
new file mode 100644
index 000000000..5d222e45d
--- /dev/null
+++ b/tools/calendar/README.md
@@ -0,0 +1,26 @@
+# Calendar server for load testing
+
+Test calendar server that provides a REST API for managing events.
+Since we may not have access to a real calendar server (such as Google Calendar API), this server will be used to test the calendar feature during load testing.
+
+Start the server like:
+```shell
+go run calendar.go --port 8083 --db ./calendar.db
+```
+
+The server uses a SQLite database to store events. This database can be modified during testing.
+
+On the fleet server, configure Google Calendar API key where `client_email` is the specified value and the `private_key` is the base URL of the calendar server:
+```json
+{
+    "client_email": "calendar-load@example.com",
+    "private_key": "http://localhost:8083"
+}
+```
+
+## Useful tricks
+
+To update all the events in SQLite database to start at the current time, do SQL query:
+```sql
+UPDATE events SET start = unixepoch('now'), end = unixepoch('now', '+30 minutes');
+```
diff --git a/tools/calendar/calendar.go b/tools/calendar/calendar.go
new file mode 100644
index 000000000..64b70c7a9
--- /dev/null
+++ b/tools/calendar/calendar.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	calendartest "github.com/fleetdm/fleet/v4/ee/server/calendar/load_test"
+	_ "github.com/mattn/go-sqlite3"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+func main() {
+	port := flag.Uint("port", 8083, "Port to listen on")
+	dbFileName := flag.String("db", "./calendar.db", "SQLite db file name")
+	flag.Parse()
+
+	handler, err := calendartest.Configure(*dbFileName)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer calendartest.Close()
+
+	listenAddr := fmt.Sprintf(":%d", *port)
+	errLogger := log.New(os.Stderr, "", log.LstdFlags)
+
+	server := &http.Server{
+		Addr:         listenAddr,
+		Handler:      handler,
+		ErrorLog:     errLogger,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 10 * time.Second,
+		IdleTimeout:  15 * time.Second,
+	}
+
+	// Start the HTTP server
+	log.Fatal(server.ListenAndServe())
+}

From 31fe9d17b97c407f41e295876b4831475494c9ba Mon Sep 17 00:00:00 2001
From: Lucas Manuel Rodriguez 
Date: Fri, 22 Mar 2024 11:20:18 -0300
Subject: [PATCH 18/36] More fixes to support users with hosts in same team and
 hosts in different teams (#17789)

#17441
---
 cmd/fleet/calendar_cron.go                |  60 +++++--
 server/datastore/mysql/calendar_events.go |  24 +++
 server/datastore/mysql/policies.go        |  21 ++-
 server/datastore/mysql/policies_test.go   | 200 ++++++++++++++++++++++
 server/fleet/datastore.go                 |   3 +-
 server/mock/datastore_mock.go             |  24 ++-
 6 files changed, 306 insertions(+), 26 deletions(-)

diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go
index fa63b487d..17962f184 100644
--- a/cmd/fleet/calendar_cron.go
+++ b/cmd/fleet/calendar_cron.go
@@ -125,7 +125,7 @@ func cronCalendarEventsForTeam(
 	for _, policy := range policies {
 		policyIDs = append(policyIDs, policy.ID)
 	}
-	hosts, err := ds.GetHostsPolicyMemberships(ctx, domain, policyIDs)
+	hosts, err := ds.GetTeamHostsPolicyMemberships(ctx, domain, team.ID, policyIDs)
 	if err != nil {
 		return fmt.Errorf("get team hosts failing policies: %w", err)
 	}
@@ -150,22 +150,28 @@ func cronCalendarEventsForTeam(
 	}
 	level.Debug(logger).Log(
 		"msg", "summary",
+		"team_id", team.ID,
 		"passing_hosts", len(passingHosts),
 		"failing_hosts", len(failingHosts),
 		"failing_hosts_without_associated_email", len(failingHostsWithoutAssociatedEmail),
 	)
 
+	// Remove calendar events from hosts that are passing the calendar policies.
+	//
+	// We execute this first to remove any calendar events for a user that is now passing
+	// policies on one of its hosts, and possibly create a new calendar event if they have
+	// another failing host on the same team.
+	if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil {
+		level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err)
+	}
+
+	// Process hosts that are failing calendar policies.
 	if err := processCalendarFailingHosts(
 		ctx, ds, calendar, orgName, failingHosts, logger,
 	); err != nil {
 		level.Info(logger).Log("msg", "processing failing hosts", "err", err)
 	}
 
-	// Remove calendar events from hosts that are passing the policies.
-	if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil {
-		level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err)
-	}
-
 	// At last we want to log the hosts that are failing and don't have an associated email.
 	logHostsWithoutAssociatedEmail(
 		domain,
@@ -184,14 +190,26 @@ func processCalendarFailingHosts(
 	hosts []fleet.HostPolicyMembershipData,
 	logger kitlog.Logger,
 ) error {
+	hosts = filterHostsWithSameEmail(hosts)
+
 	for _, host := range hosts {
 		logger := log.With(logger, "host_id", host.HostID)
 
-		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEvent(ctx, host.HostID)
+		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email)
 
 		expiredEvent := false
 		webhookAlreadyFiredThisMonth := false
 		if err == nil {
+			if hostCalendarEvent.HostID != host.HostID {
+				// This calendar event belongs to another host with this associated email,
+				// thus we skip this entry.
+				continue // continue with next host
+			}
+			if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusPending {
+				// This can happen if the host went offline (and never returned results)
+				// after setting the webhook as pending.
+				continue // continue with next host
+			}
 			now := time.Now()
 			webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent
 			if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) {
@@ -200,7 +218,7 @@ func processCalendarFailingHosts(
 				continue // continue with next host
 			}
 			webhookAlreadyFiredThisMonth = webhookAlreadyFired && sameMonth(now, calendarEvent.StartTime)
-			if calendarEvent.EndTime.Before(time.Now()) {
+			if calendarEvent.EndTime.Before(now) {
 				expiredEvent = true
 			}
 		}
@@ -232,6 +250,25 @@ func processCalendarFailingHosts(
 	return nil
 }
 
+func filterHostsWithSameEmail(hosts []fleet.HostPolicyMembershipData) []fleet.HostPolicyMembershipData {
+	minHostPerEmail := make(map[string]fleet.HostPolicyMembershipData)
+	for _, host := range hosts {
+		minHost, ok := minHostPerEmail[host.Email]
+		if !ok {
+			minHostPerEmail[host.Email] = host
+			continue
+		}
+		if host.HostID < minHost.HostID {
+			minHostPerEmail[host.Email] = host
+		}
+	}
+	filtered := make([]fleet.HostPolicyMembershipData, 0, len(minHostPerEmail))
+	for _, host := range minHostPerEmail {
+		filtered = append(filtered, host)
+	}
+	return filtered
+}
+
 func processFailingHostExistingCalendarEvent(
 	ctx context.Context,
 	ds fleet.Datastore,
@@ -416,10 +453,13 @@ func removeCalendarEventsFromPassingHosts(
 	hosts []fleet.HostPolicyMembershipData,
 ) error {
 	for _, host := range hosts {
-		calendarEvent, err := ds.GetCalendarEvent(ctx, host.Email)
+		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email)
 		switch {
 		case err == nil:
-			// OK
+			if hostCalendarEvent.HostID != host.HostID {
+				// This calendar event belongs to another host, thus we skip this entry.
+				continue
+			}
 		case fleet.IsNotFound(err):
 			continue
 		default:
diff --git a/server/datastore/mysql/calendar_events.go b/server/datastore/mysql/calendar_events.go
index 5ffc0f77f..45d8d8833 100644
--- a/server/datastore/mysql/calendar_events.go
+++ b/server/datastore/mysql/calendar_events.go
@@ -167,6 +167,30 @@ func (ds *Datastore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fl
 	return &hostCalendarEvent, &calendarEvent, nil
 }
 
+func (ds *Datastore) GetHostCalendarEventByEmail(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) {
+	const calendarEventsQuery = `
+		SELECT * FROM calendar_events WHERE email = ?
+	`
+	var calendarEvent fleet.CalendarEvent
+	if err := sqlx.GetContext(ctx, ds.reader(ctx), &calendarEvent, calendarEventsQuery, email); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, ctxerr.Wrap(ctx, notFound("CalendarEvent").WithMessage(fmt.Sprintf("email: %s", email)))
+		}
+		return nil, nil, ctxerr.Wrap(ctx, err, "get calendar event")
+	}
+	const hostCalendarEventsQuery = `
+		SELECT * FROM host_calendar_events WHERE calendar_event_id = ?
+	`
+	var hostCalendarEvent fleet.HostCalendarEvent
+	if err := sqlx.GetContext(ctx, ds.reader(ctx), &hostCalendarEvent, hostCalendarEventsQuery, calendarEvent.ID); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil, ctxerr.Wrap(ctx, notFound("HostCalendarEvent").WithID(calendarEvent.ID))
+		}
+		return nil, nil, ctxerr.Wrap(ctx, err, "get host calendar event")
+	}
+	return &hostCalendarEvent, &calendarEvent, nil
+}
+
 func (ds *Datastore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error {
 	const calendarEventsQuery = `
 		UPDATE host_calendar_events SET
diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go
index b711c8b93..71530961d 100644
--- a/server/datastore/mysql/policies.go
+++ b/server/datastore/mysql/policies.go
@@ -1172,8 +1172,12 @@ func (ds *Datastore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fl
 }
 
 // TODO(lucas): Must be tested at scale.
-// TODO(lucas): Filter out hosts with team_id == NULL
-func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
+func (ds *Datastore) GetTeamHostsPolicyMemberships(
+	ctx context.Context,
+	domain string,
+	teamID uint,
+	policyIDs []uint,
+) ([]fleet.HostPolicyMembershipData, error) {
 	query := `
 	SELECT 
 		COALESCE(sh.email, '') AS email,
@@ -1188,18 +1192,17 @@ func (ds *Datastore) GetHostsPolicyMemberships(ctx context.Context, domain strin
 		GROUP BY host_id
 	) pm
 	LEFT JOIN (
-		SELECT MIN(h.host_id) as host_id, h.email as email
-		FROM (
-			SELECT host_id, MIN(email) AS email
-			FROM host_emails WHERE email LIKE CONCAT('%@', ?)
-			GROUP BY host_id
-		) h GROUP BY h.email
+		SELECT host_id, MIN(email) AS email
+		FROM host_emails
+		JOIN hosts ON host_emails.host_id=hosts.id
+		WHERE email LIKE CONCAT('%@', ?) AND team_id = ? 
+		GROUP BY host_id
 	) sh ON sh.host_id = pm.host_id
 	JOIN hosts h ON h.id = pm.host_id
 	LEFT JOIN host_display_names hdn ON hdn.host_id = pm.host_id;
 `
 
-	query, args, err := sqlx.In(query, policyIDs, domain)
+	query, args, err := sqlx.In(query, policyIDs, domain, teamID)
 	if err != nil {
 		return nil, ctxerr.Wrapf(ctx, err, "build select get team hosts policy memberships query")
 	}
diff --git a/server/datastore/mysql/policies_test.go b/server/datastore/mysql/policies_test.go
index 514de6dd3..15ebeee17 100644
--- a/server/datastore/mysql/policies_test.go
+++ b/server/datastore/mysql/policies_test.go
@@ -60,6 +60,7 @@ func TestPolicies(t *testing.T) {
 		{"TestPoliciesNameEmoji", testPoliciesNameEmoji},
 		{"TestPoliciesNameSort", testPoliciesNameSort},
 		{"TestGetCalendarPolicies", testGetCalendarPolicies},
+		{"GetTeamHostsPolicyMemberships", testGetTeamHostsPolicyMemberships},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -2860,3 +2861,202 @@ func testGetCalendarPolicies(t *testing.T, ds *Datastore) {
 	require.Equal(t, calendarPolicies[0].ID, teamPolicy2.ID)
 	require.Equal(t, calendarPolicies[1].ID, teamPolicy3.ID)
 }
+
+func testGetTeamHostsPolicyMemberships(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+
+	team1, err := ds.NewTeam(ctx, &fleet.Team{Name: "team1"})
+	require.NoError(t, err)
+	team2, err := ds.NewTeam(ctx, &fleet.Team{Name: "team2"})
+	require.NoError(t, err)
+
+	team1Policy1, err := ds.NewTeamPolicy(ctx, team1.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 1 Policy 1",
+		Query:                 "SELECT * FROM osquery_info;",
+		CalendarEventsEnabled: true,
+	})
+	require.NoError(t, err)
+	team1Policy2, err := ds.NewTeamPolicy(ctx, team1.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 1 Policy 2",
+		Query:                 "SELECT * FROM system_info;",
+		CalendarEventsEnabled: false,
+	})
+	require.NoError(t, err)
+	team2Policy1, err := ds.NewTeamPolicy(ctx, team2.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 2 Policy 1",
+		Query:                 "SELECT * FROM os_version;",
+		CalendarEventsEnabled: true,
+	})
+	require.NoError(t, err)
+	team2Policy2, err := ds.NewTeamPolicy(ctx, team2.ID, nil, fleet.PolicyPayload{
+		Name:                  "Team 2 Policy 2",
+		Query:                 "SELECT * FROM processes;",
+		CalendarEventsEnabled: true,
+	})
+	require.NoError(t, err)
+
+	// Empty teams.
+	hostsTeam1, err := ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policy1.ID, team1Policy2.ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 0)
+
+	host1, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host1"),
+		NodeKey:        ptr.String("host1"),
+		HardwareSerial: "serial1",
+		ComputerName:   "display_name1",
+		TeamID:         &team1.ID,
+	})
+	require.NoError(t, err)
+	host2, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host2"),
+		NodeKey:        ptr.String("host2"),
+		HardwareSerial: "serial2",
+		ComputerName:   "display_name2",
+		TeamID:         &team2.ID,
+	})
+	require.NoError(t, err)
+	host3, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host3"),
+		NodeKey:        ptr.String("host3"),
+		HardwareSerial: "serial3",
+		ComputerName:   "display_name3",
+		TeamID:         &team2.ID,
+	})
+	require.NoError(t, err)
+	host4, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host4"),
+		NodeKey:        ptr.String("host4"),
+		HardwareSerial: "serial4",
+		ComputerName:   "display_name4",
+	})
+	require.NoError(t, err)
+	host5, err := ds.NewHost(ctx, &fleet.Host{
+		OsqueryHostID:  ptr.String("host5"),
+		NodeKey:        ptr.String("host5"),
+		HardwareSerial: "serial5",
+		ComputerName:   "display_name5",
+		TeamID:         &team1.ID,
+	})
+	require.NoError(t, err)
+
+	// No policy results yet.
+	hostsTeam1, err = ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policy1.ID, team1Policy2.ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 0)
+
+	err = ds.ReplaceHostDeviceMapping(ctx, host1.ID, []*fleet.HostDeviceMapping{
+		{HostID: host1.ID, Email: "foo@example.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host1.ID, []*fleet.HostDeviceMapping{
+		{HostID: host1.ID, Email: "zoo@example.com", Source: "custom"},
+	}, "custom")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host2.ID, []*fleet.HostDeviceMapping{
+		{HostID: host2.ID, Email: "foo@example.com", Source: "custom"},
+	}, "custom")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host2.ID, []*fleet.HostDeviceMapping{
+		{HostID: host2.ID, Email: "foo@other.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host3.ID, []*fleet.HostDeviceMapping{
+		{HostID: host3.ID, Email: "zoo@example.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host4.ID, []*fleet.HostDeviceMapping{
+		{HostID: host4.ID, Email: "foo@example.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+	err = ds.ReplaceHostDeviceMapping(ctx, host5.ID, []*fleet.HostDeviceMapping{
+		{HostID: host5.ID, Email: "foo@other.com", Source: "google_chrome_profiles"},
+	}, "google_chrome_profiles")
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host1, map[uint]*bool{
+		team1Policy1.ID: ptr.Bool(true),
+		team1Policy2.ID: ptr.Bool(false),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host2, map[uint]*bool{
+		team2Policy1.ID: ptr.Bool(false),
+		team2Policy2.ID: ptr.Bool(true),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host3, map[uint]*bool{
+		team2Policy1.ID: ptr.Bool(true),
+		team2Policy2.ID: ptr.Bool(true),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	err = ds.RecordPolicyQueryExecutions(ctx, host5, map[uint]*bool{
+		team1Policy1.ID: ptr.Bool(false),
+		team1Policy2.ID: ptr.Bool(false),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	team1Policies, err := ds.GetCalendarPolicies(ctx, team1.ID)
+	require.NoError(t, err)
+	require.Len(t, team1Policies, 1)
+	team2Policies, err := ds.GetCalendarPolicies(ctx, team2.ID)
+	require.NoError(t, err)
+	require.Len(t, team2Policies, 2)
+
+	hostsTeam1, err = ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policies[0].ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 2)
+	require.Equal(t, host1.ID, hostsTeam1[0].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam1[0].Email)
+	require.True(t, hostsTeam1[0].Passing)
+	require.Equal(t, "serial1", hostsTeam1[0].HostHardwareSerial)
+	require.Equal(t, "display_name1", hostsTeam1[0].HostDisplayName)
+	require.Equal(t, host5.ID, hostsTeam1[1].HostID)
+	require.Empty(t, hostsTeam1[1].Email)
+	require.False(t, hostsTeam1[1].Passing)
+	require.Equal(t, "serial5", hostsTeam1[1].HostHardwareSerial)
+	require.Equal(t, "display_name5", hostsTeam1[1].HostDisplayName)
+
+	err = ds.AddHostsToTeam(ctx, &team1.ID, []uint{host4.ID})
+	require.NoError(t, err)
+	err = ds.RecordPolicyQueryExecutions(ctx, host4, map[uint]*bool{
+		team1Policy1.ID: ptr.Bool(false),
+		team1Policy2.ID: ptr.Bool(false),
+	}, time.Now(), false)
+	require.NoError(t, err)
+
+	hostsTeam1, err = ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team1.ID, []uint{team1Policies[0].ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam1, 3)
+	require.Equal(t, host1.ID, hostsTeam1[0].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam1[0].Email)
+	require.True(t, hostsTeam1[0].Passing)
+	require.Equal(t, "serial1", hostsTeam1[0].HostHardwareSerial)
+	require.Equal(t, "display_name1", hostsTeam1[0].HostDisplayName)
+	require.Equal(t, host4.ID, hostsTeam1[1].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam1[1].Email)
+	require.False(t, hostsTeam1[1].Passing)
+	require.Equal(t, "serial4", hostsTeam1[1].HostHardwareSerial)
+	require.Equal(t, "display_name4", hostsTeam1[1].HostDisplayName)
+	require.Equal(t, host5.ID, hostsTeam1[2].HostID)
+	require.Empty(t, hostsTeam1[2].Email)
+	require.False(t, hostsTeam1[2].Passing)
+	require.Equal(t, "serial5", hostsTeam1[2].HostHardwareSerial)
+	require.Equal(t, "display_name5", hostsTeam1[2].HostDisplayName)
+
+	hostsTeam2, err := ds.GetTeamHostsPolicyMemberships(ctx, "example.com", team2.ID, []uint{team2Policies[0].ID, team2Policies[1].ID})
+	require.NoError(t, err)
+	require.Len(t, hostsTeam2, 2)
+	require.Equal(t, host2.ID, hostsTeam2[0].HostID)
+	require.Equal(t, "foo@example.com", hostsTeam2[0].Email)
+	require.False(t, hostsTeam2[0].Passing)
+	require.Equal(t, "serial2", hostsTeam2[0].HostHardwareSerial)
+	require.Equal(t, "display_name2", hostsTeam2[0].HostDisplayName)
+	require.Equal(t, host3.ID, hostsTeam2[1].HostID)
+	require.Equal(t, "zoo@example.com", hostsTeam2[1].Email)
+	require.True(t, hostsTeam2[1].Passing)
+	require.Equal(t, "serial3", hostsTeam2[1].HostHardwareSerial)
+	require.Equal(t, "display_name3", hostsTeam2[1].HostDisplayName)
+}
diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go
index 65098efdf..a2f8bf6cd 100644
--- a/server/fleet/datastore.go
+++ b/server/fleet/datastore.go
@@ -594,7 +594,7 @@ type Datastore interface {
 
 	PolicyQueriesForHost(ctx context.Context, host *Host) (map[string]string, error)
 
-	GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]HostPolicyMembershipData, error)
+	GetTeamHostsPolicyMemberships(ctx context.Context, domain string, teamID uint, policyIDs []uint) ([]HostPolicyMembershipData, error)
 	GetCalendarPolicies(ctx context.Context, teamID uint) ([]PolicyCalendarData, error)
 
 	// Methods used for async processing of host policy query results.
@@ -624,6 +624,7 @@ type Datastore interface {
 	DeleteCalendarEvent(ctx context.Context, calendarEventID uint) error
 	UpdateCalendarEvent(ctx context.Context, calendarEventID uint, startTime time.Time, endTime time.Time, data []byte) error
 	GetHostCalendarEvent(ctx context.Context, hostID uint) (*HostCalendarEvent, *CalendarEvent, error)
+	GetHostCalendarEventByEmail(ctx context.Context, email string) (*HostCalendarEvent, *CalendarEvent, error)
 	UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status CalendarWebhookStatus) error
 	ListCalendarEvents(ctx context.Context, teamID *uint) ([]*CalendarEvent, error)
 	ListOutOfDateCalendarEvents(ctx context.Context, t time.Time) ([]*CalendarEvent, error)
diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go
index 1b77b29cb..425e0945e 100644
--- a/server/mock/datastore_mock.go
+++ b/server/mock/datastore_mock.go
@@ -440,7 +440,7 @@ type UpdateHostPolicyCountsFunc func(ctx context.Context) error
 
 type PolicyQueriesForHostFunc func(ctx context.Context, host *fleet.Host) (map[string]string, error)
 
-type GetHostsPolicyMembershipsFunc func(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error)
+type GetTeamHostsPolicyMembershipsFunc func(ctx context.Context, domain string, teamID uint, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error)
 
 type GetCalendarPoliciesFunc func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error)
 
@@ -472,6 +472,8 @@ type UpdateCalendarEventFunc func(ctx context.Context, calendarEventID uint, sta
 
 type GetHostCalendarEventFunc func(ctx context.Context, hostID uint) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error)
 
+type GetHostCalendarEventByEmailFunc func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error)
+
 type UpdateHostCalendarWebhookStatusFunc func(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error
 
 type ListCalendarEventsFunc func(ctx context.Context, teamID *uint) ([]*fleet.CalendarEvent, error)
@@ -1512,8 +1514,8 @@ type DataStore struct {
 	PolicyQueriesForHostFunc        PolicyQueriesForHostFunc
 	PolicyQueriesForHostFuncInvoked bool
 
-	GetHostsPolicyMembershipsFunc        GetHostsPolicyMembershipsFunc
-	GetHostsPolicyMembershipsFuncInvoked bool
+	GetTeamHostsPolicyMembershipsFunc        GetTeamHostsPolicyMembershipsFunc
+	GetTeamHostsPolicyMembershipsFuncInvoked bool
 
 	GetCalendarPoliciesFunc        GetCalendarPoliciesFunc
 	GetCalendarPoliciesFuncInvoked bool
@@ -1560,6 +1562,9 @@ type DataStore struct {
 	GetHostCalendarEventFunc        GetHostCalendarEventFunc
 	GetHostCalendarEventFuncInvoked bool
 
+	GetHostCalendarEventByEmailFunc        GetHostCalendarEventByEmailFunc
+	GetHostCalendarEventByEmailFuncInvoked bool
+
 	UpdateHostCalendarWebhookStatusFunc        UpdateHostCalendarWebhookStatusFunc
 	UpdateHostCalendarWebhookStatusFuncInvoked bool
 
@@ -3649,11 +3654,11 @@ func (s *DataStore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host)
 	return s.PolicyQueriesForHostFunc(ctx, host)
 }
 
-func (s *DataStore) GetHostsPolicyMemberships(ctx context.Context, domain string, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
+func (s *DataStore) GetTeamHostsPolicyMemberships(ctx context.Context, domain string, teamID uint, policyIDs []uint) ([]fleet.HostPolicyMembershipData, error) {
 	s.mu.Lock()
-	s.GetHostsPolicyMembershipsFuncInvoked = true
+	s.GetTeamHostsPolicyMembershipsFuncInvoked = true
 	s.mu.Unlock()
-	return s.GetHostsPolicyMembershipsFunc(ctx, domain, policyIDs)
+	return s.GetTeamHostsPolicyMembershipsFunc(ctx, domain, teamID, policyIDs)
 }
 
 func (s *DataStore) GetCalendarPolicies(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) {
@@ -3761,6 +3766,13 @@ func (s *DataStore) GetHostCalendarEvent(ctx context.Context, hostID uint) (*fle
 	return s.GetHostCalendarEventFunc(ctx, hostID)
 }
 
+func (s *DataStore) GetHostCalendarEventByEmail(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) {
+	s.mu.Lock()
+	s.GetHostCalendarEventByEmailFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetHostCalendarEventByEmailFunc(ctx, email)
+}
+
 func (s *DataStore) UpdateHostCalendarWebhookStatus(ctx context.Context, hostID uint, status fleet.CalendarWebhookStatus) error {
 	s.mu.Lock()
 	s.UpdateHostCalendarWebhookStatusFuncInvoked = true

From c6e2e8d6c42493ab985fc5a2aac07c9609cf1f75 Mon Sep 17 00:00:00 2001
From: Lucas Manuel Rodriguez 
Date: Fri, 22 Mar 2024 14:16:08 -0300
Subject: [PATCH 19/36] Always create event next 3rd Tuesday (#17799)

Fix to always create events for next 3rd Tuesday #17441
---
 cmd/fleet/calendar_cron.go      |  32 +++------
 cmd/fleet/calendar_cron_test.go | 113 +++++++++++++++++++-------------
 2 files changed, 77 insertions(+), 68 deletions(-)

diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go
index 17962f184..238260ea3 100644
--- a/cmd/fleet/calendar_cron.go
+++ b/cmd/fleet/calendar_cron.go
@@ -198,7 +198,6 @@ func processCalendarFailingHosts(
 		hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email)
 
 		expiredEvent := false
-		webhookAlreadyFiredThisMonth := false
 		if err == nil {
 			if hostCalendarEvent.HostID != host.HostID {
 				// This calendar event belongs to another host with this associated email,
@@ -217,7 +216,6 @@ func processCalendarFailingHosts(
 				// we give a grace period of one day for the host before we schedule a new event.
 				continue // continue with next host
 			}
-			webhookAlreadyFiredThisMonth = webhookAlreadyFired && sameMonth(now, calendarEvent.StartTime)
 			if calendarEvent.EndTime.Before(now) {
 				expiredEvent = true
 			}
@@ -237,7 +235,7 @@ func processCalendarFailingHosts(
 			}
 		case fleet.IsNotFound(err) || expiredEvent:
 			if err := processFailingHostCreateCalendarEvent(
-				ctx, ds, userCalendar, orgName, host, webhookAlreadyFiredThisMonth,
+				ctx, ds, userCalendar, orgName, host,
 			); err != nil {
 				level.Info(logger).Log("msg", "process failing host create calendar event", "err", err)
 				continue // continue with next host
@@ -357,21 +355,14 @@ func sameDate(t1 time.Time, t2 time.Time) bool {
 	return y1 == y2 && m1 == m2 && d1 == d2
 }
 
-func sameMonth(t1 time.Time, t2 time.Time) bool {
-	y1, m1, _ := t1.Date()
-	y2, m2, _ := t2.Date()
-	return y1 == y2 && m1 == m2
-}
-
 func processFailingHostCreateCalendarEvent(
 	ctx context.Context,
 	ds fleet.Datastore,
 	userCalendar fleet.UserCalendar,
 	orgName string,
 	host fleet.HostPolicyMembershipData,
-	webhookAlreadyFiredThisMonth bool,
 ) error {
-	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar, webhookAlreadyFiredThisMonth)
+	calendarEvent, err := attemptCreatingEventOnUserCalendar(orgName, host, userCalendar)
 	if err != nil {
 		return fmt.Errorf("create event on user calendar: %w", err)
 	}
@@ -385,10 +376,9 @@ func attemptCreatingEventOnUserCalendar(
 	orgName string,
 	host fleet.HostPolicyMembershipData,
 	userCalendar fleet.UserCalendar,
-	webhookAlreadyFiredThisMonth bool,
 ) (*fleet.CalendarEvent, error) {
 	year, month, today := time.Now().Date()
-	preferredDate := getPreferredCalendarEventDate(year, month, today, webhookAlreadyFiredThisMonth)
+	preferredDate := getPreferredCalendarEventDate(year, month, today)
 	for {
 		calendarEvent, err := userCalendar.CreateEvent(
 			preferredDate, func(conflict bool) string {
@@ -408,10 +398,7 @@ func attemptCreatingEventOnUserCalendar(
 	}
 }
 
-func getPreferredCalendarEventDate(
-	year int, month time.Month, today int,
-	webhookAlreadyFired bool,
-) time.Time {
+func getPreferredCalendarEventDate(year int, month time.Month, today int) time.Time {
 	const (
 		// 3rd Tuesday of Month
 		preferredWeekDay = time.Tuesday
@@ -425,12 +412,13 @@ func getPreferredCalendarEventDate(
 	}
 	preferredDate := firstDayOfMonth.AddDate(0, 0, offset+(7*(preferredOrdinal-1)))
 	if today > preferredDate.Day() {
-		today_ := time.Date(year, month, today, 0, 0, 0, 0, time.UTC)
-		if webhookAlreadyFired {
-			nextMonth := today_.AddDate(0, 1, 0) // move to next month
-			return getPreferredCalendarEventDate(nextMonth.Year(), nextMonth.Month(), 1, false)
+		// We are past the preferred date, so we move to next month and calculate again.
+		month := month + 1
+		if month == 13 {
+			month = 1
+			year += 1
 		}
-		preferredDate = addBusinessDay(today_)
+		return getPreferredCalendarEventDate(year, month, 1)
 	}
 	return preferredDate
 }
diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go
index 905b79c87..84f5ece52 100644
--- a/cmd/fleet/calendar_cron_test.go
+++ b/cmd/fleet/calendar_cron_test.go
@@ -12,72 +12,93 @@ func TestGetPreferredCalendarEventDate(t *testing.T) {
 		return time.Date(year, month, day, 0, 0, 0, 0, time.UTC)
 	}
 	for _, tc := range []struct {
-		name                  string
-		year                  int
-		month                 time.Month
-		daysStart             int
-		daysEnd               int
-		webhookFiredThisMonth bool
+		name      string
+		year      int
+		month     time.Month
+		daysStart int
+		daysEnd   int
 
 		expected time.Time
 	}{
 		{
-			name:                  "March 2024 (webhook hasn't fired)",
-			year:                  2024,
-			month:                 3,
-			daysStart:             1,
-			daysEnd:               31,
-			webhookFiredThisMonth: false,
+			name:      "March 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     3,
+			daysStart: 1,
+			daysEnd:   19,
 
 			expected: date(2024, 3, 19),
 		},
 		{
-			name:                  "March 2024 (webhook has fired, days before 3rd Tuesday)",
-			year:                  2024,
-			month:                 3,
-			daysStart:             1,
-			daysEnd:               18,
-			webhookFiredThisMonth: true,
-
-			expected: date(2024, 3, 19),
-		},
-		{
-			name:                  "March 2024 (webhook has fired, days after 3rd Tuesday)",
-			year:                  2024,
-			month:                 3,
-			daysStart:             20,
-			daysEnd:               30,
-			webhookFiredThisMonth: true,
+			name:      "March 2024 (past 3rd Tuesday)",
+			year:      2024,
+			month:     3,
+			daysStart: 20,
+			daysEnd:   31,
 
 			expected: date(2024, 4, 16),
 		},
 		{
-			name:                  "April 2024 (webhook hasn't fired)",
-			year:                  2024,
-			month:                 4,
-			daysEnd:               30,
-			webhookFiredThisMonth: false,
+			name:      "April 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     4,
+			daysStart: 1,
+			daysEnd:   16,
 
 			expected: date(2024, 4, 16),
 		},
+		{
+			name:      "April 2024 (after 3rd Tuesday)",
+			year:      2024,
+			month:     4,
+			daysStart: 17,
+			daysEnd:   30,
+
+			expected: date(2024, 5, 21),
+		},
+		{
+			name:      "May 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     5,
+			daysStart: 1,
+			daysEnd:   21,
+
+			expected: date(2024, 5, 21),
+		},
+		{
+			name:      "May 2024 (after 3rd Tuesday)",
+			year:      2024,
+			month:     5,
+			daysStart: 22,
+			daysEnd:   31,
+
+			expected: date(2024, 6, 18),
+		},
+		{
+			name:      "Dec 2024 (before 3rd Tuesday)",
+			year:      2024,
+			month:     12,
+			daysStart: 1,
+			daysEnd:   17,
+
+			expected: date(2024, 12, 17),
+		},
+		{
+			name:      "Dec 2024 (after 3rd Tuesday)",
+			year:      2024,
+			month:     12,
+			daysStart: 18,
+			daysEnd:   31,
+
+			expected: date(2025, 1, 21),
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			for day := tc.daysStart; day <= tc.daysEnd; day++ {
-				actual := getPreferredCalendarEventDate(tc.year, tc.month, day, tc.webhookFiredThisMonth)
+				actual := getPreferredCalendarEventDate(tc.year, tc.month, day)
 				require.NotEqual(t, actual.Weekday(), time.Saturday)
 				require.NotEqual(t, actual.Weekday(), time.Sunday)
-				if day <= tc.expected.Day() || tc.webhookFiredThisMonth {
-					require.Equal(t, tc.expected, actual)
-				} else {
-					today := date(tc.year, tc.month, day)
-					if weekday := today.Weekday(); weekday == time.Friday {
-						require.Equal(t, today.AddDate(0, 0, +3), actual)
-					} else if weekday == time.Saturday {
-						require.Equal(t, today.AddDate(0, 0, +2), actual)
-					} else {
-						require.Equal(t, today.AddDate(0, 0, +1), actual)
-					}
-				}
+				require.Equal(t, tc.expected, actual)
 			}
 		})
 	}

From fbb271caeefb242c182bad2d51163adbedc174b1 Mon Sep 17 00:00:00 2001
From: RachelElysia <71795832+RachelElysia@users.noreply.github.com>
Date: Fri, 22 Mar 2024 14:52:54 -0400
Subject: [PATCH 20/36] Fleet UI: Calendar settings iterations (#17779)

---
 .../cards/Calendars/Calendars.tsx             | 33 +++++++++++++++++--
 .../cards/Calendars/_styles.scss              |  5 +++
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
index de7c79a13..da22ea480 100644
--- a/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
+++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/Calendars.tsx
@@ -33,7 +33,7 @@ const API_KEY_JSON_PLACEHOLDER = `{
   "type": "service_account",
   "project_id": "fleet-in-your-calendar",
   "private_key_id": "",
-  "private_key": "-----BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY-----\n",
+  "private_key": "-----BEGIN PRIVATE KEY----\\n\\n-----END PRIVATE KEY-----\\n",
   "client_email": "fleet-calendar-events@fleet-in-your-calendar.iam.gserviceaccount.com",
   "client_id": "",
   "auth_uri": "https://accounts.google.com/o/oauth2/auth",
@@ -58,6 +58,16 @@ interface ICalendarsFormData {
   apiKeyJson?: string;
 }
 
+// Used to surface error.message in UI of unknown error type
+type ErrorWithMessage = {
+  message: string;
+  [key: string]: unknown;
+};
+
+const isErrorWithMessage = (error: unknown): error is ErrorWithMessage => {
+  return (error as ErrorWithMessage).message !== undefined;
+};
+
 const baseClass = "calendars-integration";
 
 const Calendars = (): JSX.Element => {
@@ -103,7 +113,18 @@ const Calendars = (): JSX.Element => {
       errors.apiKeyJson = "API key JSON must be present";
     }
     if (!curFormData.domain && !!curFormData.apiKeyJson) {
-      errors.apiKeyJson = "Domain must be present";
+      errors.domain = "Domain must be present";
+    }
+    if (curFormData.apiKeyJson) {
+      try {
+        JSON.parse(curFormData.apiKeyJson);
+      } catch (e: unknown) {
+        if (isErrorWithMessage(e)) {
+          errors.apiKeyJson = e.message.toString();
+        } else {
+          throw e;
+        }
+      }
     }
     return errors;
   };
@@ -277,6 +298,7 @@ const Calendars = (): JSX.Element => {
                     placeholder={API_KEY_JSON_PLACEHOLDER}
                     ignore1password
                     inputClassName={`${baseClass}__api-key-json`}
+                    error={formErrors.apiKeyJson}
                   />
                    {
                         />
                       
                     }
+                    error={formErrors.domain}
                   />
); diff --git a/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss index 7045a443e..01db771e0 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss +++ b/frontend/pages/admin/IntegrationsPage/cards/Calendars/_styles.scss @@ -4,6 +4,10 @@ color: $core-fleet-black; } + p { + margin: $pad-large 0; + } + ui { margin-block-start: $pad-small; } @@ -30,6 +34,7 @@ #oauth-scopes { font-family: "SourceCodePro", $monospace; + color: $core-fleet-black; min-height: 80px; padding: $pad-medium; padding-right: $pad-xxlarge; From a10aac29c609b4c930a7474bfdb2a67c45943d1b Mon Sep 17 00:00:00 2001 From: Jacob Shandling <61553566+jacobshandling@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:54:32 -0700 Subject: [PATCH 21/36] =?UTF-8?q?UI=20=E2=80=93=20Calendar=20events=20moda?= =?UTF-8?q?l=20follow=20up=20(#17788)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Follow-up work to #17717 **Finalize disabled options and tooltips:** Screenshot 2024-03-21 at 5 14 40 PM Screenshot 2024-03-21 at 5 15 13 PM **Only update policies and settings when there's a diff:** ![1(1)](https://github.com/fleetdm/fleet/assets/61553566/183d1834-3c54-4fef-a208-dfbb0354e507) **Reorganize onChange handlers, types** - [x] Manual QA for all new/changed functionality --------- Co-authored-by: Jacob Shandling --- frontend/interfaces/integration.ts | 8 +- frontend/pages/SoftwarePage/SoftwarePage.tsx | 6 +- .../AddIntegrationModal.tsx | 4 +- .../EditIntegrationModal.tsx | 4 +- .../IntegrationForm/IntegrationForm.tsx | 4 +- .../HostActionsDropdown/_styles.scss | 1 + .../ManagePoliciesPage/ManagePoliciesPage.tsx | 89 ++++++++++++------- .../policies/ManagePoliciesPage/_styles.scss | 30 ++++++- .../CalendarEventsModal.tsx | 54 ++++++----- .../OtherWorkflowsModal.tsx | 7 +- frontend/styles/var/mixins.scss | 1 - 11 files changed, 132 insertions(+), 76 deletions(-) diff --git a/frontend/interfaces/integration.ts b/frontend/interfaces/integration.ts index aea79f99e..f6302a67b 100644 --- a/frontend/interfaces/integration.ts +++ b/frontend/interfaces/integration.ts @@ -75,15 +75,17 @@ interface ITeamCalendarSettings { // separated – it can be present without the other 2 without nullifying them. // TODO: Update these types to reflect this. -export interface IIntegrations { +export interface IZendeskJiraIntegrations { zendesk: IZendeskIntegration[]; jira: IJiraIntegration[]; } -export interface IGlobalIntegrations extends IIntegrations { +// reality is that IZendeskJiraIntegrations are optional – should be something like `extends +// Partial`, but that leads to a mess of types to resolve. +export interface IGlobalIntegrations extends IZendeskJiraIntegrations { google_calendar?: IGlobalCalendarIntegration[] | null; } -export interface ITeamIntegrations extends IIntegrations { +export interface ITeamIntegrations extends IZendeskJiraIntegrations { google_calendar?: ITeamCalendarSettings | null; } diff --git a/frontend/pages/SoftwarePage/SoftwarePage.tsx b/frontend/pages/SoftwarePage/SoftwarePage.tsx index 0d513c348..20ccb325e 100644 --- a/frontend/pages/SoftwarePage/SoftwarePage.tsx +++ b/frontend/pages/SoftwarePage/SoftwarePage.tsx @@ -11,7 +11,7 @@ import { import { IJiraIntegration, IZendeskIntegration, - IIntegrations, + IZendeskJiraIntegrations, } from "interfaces/integration"; import { ITeamConfig } from "interfaces/team"; import { IWebhookSoftwareVulnerabilities } from "interfaces/webhook"; @@ -186,7 +186,9 @@ const SoftwarePage = ({ children, router, location }: ISoftwarePageProps) => { const vulnWebhookSettings = softwareConfig?.webhook_settings?.vulnerabilities_webhook; const isVulnWebhookEnabled = !!vulnWebhookSettings?.enable_vulnerabilities_webhook; - const isVulnIntegrationEnabled = (integrations?: IIntegrations) => { + const isVulnIntegrationEnabled = ( + integrations?: IZendeskJiraIntegrations + ) => { return ( !!integrations?.jira?.some((j) => j.enable_software_vulnerabilities) || !!integrations?.zendesk?.some((z) => z.enable_software_vulnerabilities) diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx index 0dc4dc630..ef3a69322 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx +++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/AddIntegrationModal/AddIntegrationModal.tsx @@ -4,7 +4,7 @@ import Modal from "components/Modal"; // @ts-ignore import Dropdown from "components/forms/fields/Dropdown"; import CustomLink from "components/CustomLink"; -import { IIntegration, IIntegrations } from "interfaces/integration"; +import { IIntegration, IZendeskJiraIntegrations } from "interfaces/integration"; import IntegrationForm from "../IntegrationForm"; const baseClass = "add-integration-modal"; @@ -17,7 +17,7 @@ interface IAddIntegrationModalProps { ) => void; serverErrors?: { base: string; email: string }; backendValidators: { [key: string]: string }; - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; testingConnection: boolean; } diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx index 83d99a14f..e5219f270 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx +++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/EditIntegrationModal/EditIntegrationModal.tsx @@ -4,7 +4,7 @@ import Modal from "components/Modal"; import Spinner from "components/Spinner"; import { IIntegration, - IIntegrations, + IZendeskJiraIntegrations, IIntegrationTableData, } from "interfaces/integration"; import IntegrationForm from "../IntegrationForm"; @@ -15,7 +15,7 @@ interface IEditIntegrationModalProps { onCancel: () => void; onSubmit: (jiraIntegrationSubmitData: IIntegration[]) => void; backendValidators: { [key: string]: string }; - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; integrationEditing?: IIntegrationTableData; testingConnection: boolean; } diff --git a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx index 0cce5bb5a..1d4bad995 100644 --- a/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx +++ b/frontend/pages/admin/IntegrationsPage/cards/Integrations/components/IntegrationForm/IntegrationForm.tsx @@ -5,7 +5,7 @@ import { IIntegrationFormData, IIntegrationTableData, IIntegration, - IIntegrations, + IZendeskJiraIntegrations, IIntegrationType, } from "interfaces/integration"; @@ -26,7 +26,7 @@ interface IIntegrationFormProps { integrationDestination: string ) => void; integrationEditing?: IIntegrationTableData; - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; integrationEditingUrl?: string; integrationEditingUsername?: string; integrationEditingEmail?: string; diff --git a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss index 06bd48a65..04d394a3b 100644 --- a/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss +++ b/frontend/pages/hosts/details/HostDetailsPage/HostActionsDropdown/_styles.scss @@ -1,5 +1,6 @@ .host-actions-dropdown { @include button-dropdown; + color: $core-fleet-black; .Select-multi-value-wrapper { width: 55px; } diff --git a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx index 80a832026..a49d27612 100644 --- a/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/ManagePoliciesPage.tsx @@ -14,7 +14,7 @@ import { TableContext } from "context/table"; import { NotificationContext } from "context/notification"; import useTeamIdParam from "hooks/useTeamIdParam"; import { IConfig, IWebhookSettings } from "interfaces/config"; -import { IIntegrations } from "interfaces/integration"; +import { IZendeskJiraIntegrations } from "interfaces/integration"; import { IPolicyStats, ILoadAllPoliciesResponse, @@ -519,10 +519,9 @@ const ManagePolicyPage = ({ router?.replace(locationPath); }; - const handleUpdateAutomations = async (requestBody: { + const handleUpdateOtherWorkflows = async (requestBody: { webhook_settings: Pick; - // TODO - update below type to specify team integration - integrations: IIntegrations; + integrations: IZendeskJiraIntegrations; }) => { setIsUpdatingAutomations(true); try { @@ -549,32 +548,52 @@ const ManagePolicyPage = ({ setUpdatingPolicyEnabledCalendarEvents(true); try { - // update enabled and URL in config - const configResponse = teamsAPI.update( - { - integrations: { - google_calendar: { - enable_calendar_events: formData.enabled, - webhook_url: formData.url, + // update team config if either field has been changed + const responses: Promise[] = []; + if ( + formData.enabled !== + teamConfig?.integrations.google_calendar?.enable_calendar_events || + formData.url !== teamConfig?.integrations.google_calendar?.webhook_url + ) { + responses.push( + teamsAPI.update( + { + integrations: { + google_calendar: { + enable_calendar_events: formData.enabled, + webhook_url: formData.url, + }, + // These fields will never actually be changed here. See comment above + // IGlobalIntegrations definition. + zendesk: teamConfig?.integrations.zendesk || [], + jira: teamConfig?.integrations.jira || [], + }, }, - // TODO - can omit these? - zendesk: teamConfig?.integrations.zendesk || [], - jira: teamConfig?.integrations.jira || [], - }, - }, - teamIdForApi - ); + teamIdForApi + ) + ); + } - // update policies calendar events enabled - // TODO - only update changed policies - const policyResponses = formData.policies.map((formPolicy) => - teamPoliciesAPI.update(formPolicy.id, { - calendar_events_enabled: formPolicy.isChecked, - team_id: teamIdForApi, + // update changed policies calendar events enabled + const changedPolicies = formData.policies.filter((formPolicy) => { + const prevPolicyState = teamPolicies?.find( + (policy) => policy.id === formPolicy.id + ); + return ( + formPolicy.isChecked !== prevPolicyState?.calendar_events_enabled + ); + }); + + responses.concat( + changedPolicies.map((changedPolicy) => { + return teamPoliciesAPI.update(changedPolicy.id, { + calendar_events_enabled: changedPolicy.isChecked, + team_id: teamIdForApi, + }); }) ); - await Promise.all([configResponse, ...policyResponses]); + await Promise.all(responses); renderFlash("success", "Successfully updated policy automations."); } catch { renderFlash( @@ -761,8 +780,16 @@ const ManagePolicyPage = ({ const tipId = uniqueId(); calEventsLabel = ( -
Calendar events
- +
+ Calendar events +
+ Available in Fleet Premium @@ -771,13 +798,15 @@ const ManagePolicyPage = ({ const tipId = uniqueId(); calEventsLabel = ( -
Calendar events
+
+ Calendar events +
Select a team to manage
@@ -920,7 +949,7 @@ const ManagePolicyPage = ({ availablePolicies={availablePoliciesForAutomation} isUpdatingAutomations={isUpdatingAutomations} onExit={toggleOtherWorkflowsModal} - handleSubmit={handleUpdateAutomations} + handleSubmit={handleUpdateOtherWorkflows} /> )} {showAddPolicyModal && ( diff --git a/frontend/pages/policies/ManagePoliciesPage/_styles.scss b/frontend/pages/policies/ManagePoliciesPage/_styles.scss index ed99ad013..3c88a2db1 100644 --- a/frontend/pages/policies/ManagePoliciesPage/_styles.scss +++ b/frontend/pages/policies/ManagePoliciesPage/_styles.scss @@ -21,19 +21,43 @@ .Select > .Select-menu-outer { left: -186px; width: 360px; + .dropdown__help-text { + color: $ui-fleet-black-50; + } .is-disabled * { color: $ui-fleet-black-25; + .label-text { + font-style: normal; + // increase height to allow for broader tooltip activation area + position: absolute; + height: 34px; + width: 100%; + } + .dropdown__help-text { + // compensate for absolute label-text height + margin-top: 20px; + } .react-tooltip { @include tooltip-text; + font-style: normal; + text-align: center; } } } .Select-control { margin-top: 0; gap: 6px; - } - .Select-placeholder { - font-weight: $bold; + .Select-placeholder { + color: $core-vibrant-blue; + font-weight: $bold; + } + .dropdown__custom-arrow .dropdown__icon { + svg { + path { + stroke: $core-vibrant-blue-over; + } + } + } } } diff --git a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx index eba5abb4e..93847411e 100644 --- a/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx +++ b/frontend/pages/policies/ManagePoliciesPage/components/CalendarEventsModal/CalendarEventsModal.tsx @@ -55,10 +55,6 @@ const CalendarEventsModal = ({ const [formData, setFormData] = useState({ enabled, url, - // TODO - stay udpdated on state of backend approach to syncing policies in the policies table - // and in the new calendar table - // id may change if policy was deleted - // name could change if policy was renamed policies: policies.map((policy) => ({ name: policy.name, id: policy.id, @@ -87,29 +83,26 @@ const CalendarEventsModal = ({ return errors; }; - // TODO - separate change handlers for checkboxes: - // const onPolicyUpdate = ... - // const onTextFieldUpdate = ... - - const onInputChange = useCallback( - (newVal: { name: FormNames; value: string | number | boolean }) => { + // two onChange handlers to handle different levels of nesting in the form data + const onFeatureEnabledOrUrlChange = useCallback( + (newVal: { name: "enabled" | "url"; value: string | boolean }) => { const { name, value } = newVal; - let newFormData: ICalendarEventsFormData; - // for the first two fields, set the new value directly - if (["enabled", "url"].includes(name)) { - newFormData = { ...formData, [name]: value }; - } else if (typeof value === "boolean") { - // otherwise, set the value for a nested policy - const newFormPolicies = formData.policies.map((formPolicy) => { - if (formPolicy.name === name) { - return { ...formPolicy, isChecked: value }; - } - return formPolicy; - }); - newFormData = { ...formData, policies: newFormPolicies }; - } else { - throw TypeError("Unexpected value type for policy checkbox"); - } + const newFormData = { ...formData, [name]: value }; + setFormData(newFormData); + setFormErrors(validateCalendarEventsFormData(newFormData)); + }, + [formData] + ); + const onPolicyEnabledChange = useCallback( + (newVal: { name: FormNames; value: boolean }) => { + const { name, value } = newVal; + const newFormPolicies = formData.policies.map((formPolicy) => { + if (formPolicy.name === name) { + return { ...formPolicy, isChecked: value }; + } + return formPolicy; + }); + const newFormData = { ...formData, policies: newFormPolicies }; setFormData(newFormData); setFormErrors(validateCalendarEventsFormData(newFormData)); }, @@ -157,7 +150,7 @@ const CalendarEventsModal = ({ name={name} // can't use parseTarget as value needs to be set to !currentValue onChange={() => { - onInputChange({ name, value: !isChecked }); + onPolicyEnabledChange({ name, value: !isChecked }); }} > {name} @@ -232,7 +225,10 @@ const CalendarEventsModal = ({ { - onInputChange({ name: "enabled", value: !formData.enabled }); + onFeatureEnabledOrUrlChange({ + name: "enabled", + value: !formData.enabled, + }); }} inactiveText="Disabled" activeText="Enabled" @@ -251,7 +247,7 @@ const CalendarEventsModal = ({ { +const findEnabledIntegration = ({ + jira, + zendesk, +}: IZendeskJiraIntegrations) => { return ( jira?.find((j) => j.enable_failing_policies) || zendesk?.find((z) => z.enable_failing_policies) diff --git a/frontend/styles/var/mixins.scss b/frontend/styles/var/mixins.scss index 3751ffadf..fab5ee9ec 100644 --- a/frontend/styles/var/mixins.scss +++ b/frontend/styles/var/mixins.scss @@ -275,7 +275,6 @@ $max-width: 2560px; } .Select-placeholder { - color: $core-fleet-black; font-size: 14px; line-height: normal; padding-left: 0; From 62049b04bd79ef16a97f6e38e43bf30fe01f6cd9 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 22 Mar 2024 14:25:03 -0500 Subject: [PATCH 22/36] Added TestEventForDifferentHost for calendar_cron. (#17802) Added TestEventForDifferentHost for calendar_cron. --- cmd/fleet/calendar_cron_test.go | 87 +++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go index 84f5ece52..c6db483c5 100644 --- a/cmd/fleet/calendar_cron_test.go +++ b/cmd/fleet/calendar_cron_test.go @@ -1,6 +1,11 @@ package main import ( + "context" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + kitlog "github.com/go-kit/log" + "os" "testing" "time" @@ -8,6 +13,7 @@ import ( ) func TestGetPreferredCalendarEventDate(t *testing.T) { + t.Parallel() date := func(year int, month time.Month, day int) time.Time { return time.Date(year, month, day, 0, 0, 0, 0, time.UTC) } @@ -103,3 +109,84 @@ func TestGetPreferredCalendarEventDate(t *testing.T) { }) } } + +// TestEventForDifferentHost tests case when event exists, but for a different host. Nothing should happen. +// The old event will eventually be cleaned up by the cleanup job, and afterward a new event will be created. +func TestEventForDifferentHost(t *testing.T) { + t.Parallel() + ds := new(mock.Store) + ctx := context.Background() + logger := kitlog.With(kitlog.NewLogfmtLogger(os.Stdout)) + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + {}, + }, + }, + }, nil + } + teamID1 := uint(1) + ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { + return []*fleet.Team{ + { + ID: teamID1, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + }, + }, + }, + }, + }, nil + } + policyID1 := uint(10) + ds.GetCalendarPoliciesFunc = func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + require.Equal(t, teamID1, teamID) + return []fleet.PolicyCalendarData{ + { + ID: policyID1, + Name: "Policy 1", + }, + }, nil + } + hostID1 := uint(100) + hostID2 := uint(101) + userEmail1 := "user@example.com" + ds.GetTeamHostsPolicyMembershipsFunc = func( + ctx context.Context, domain string, teamID uint, policyIDs []uint, + ) ([]fleet.HostPolicyMembershipData, error) { + require.Equal(t, teamID1, teamID) + require.Equal(t, []uint{policyID1}, policyIDs) + return []fleet.HostPolicyMembershipData{ + { + HostID: hostID1, + Email: userEmail1, + Passing: false, + }, + }, nil + } + // Return an existing event, but for a different host + eventTime := time.Now().Add(time.Hour) + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + require.Equal(t, userEmail1, email) + calEvent := &fleet.CalendarEvent{ + ID: 1, + Email: email, + StartTime: eventTime, + EndTime: eventTime, + } + hcEvent := &fleet.HostCalendarEvent{ + ID: 1, + HostID: hostID2, + CalendarEventID: 1, + WebhookStatus: fleet.CalendarWebhookStatusNone, + } + return hcEvent, calEvent, nil + } + + err := cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) + +} From 355379aa0b9050f5588c10693d0a88920f9b88ab Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Fri, 22 Mar 2024 16:26:11 -0300 Subject: [PATCH 23/36] Fleet calendar process 100 hosts at a time (#17806) Add concurrency for #17441. --- cmd/fleet/calendar_cron.go | 190 +++++++++++++++++++++++-------------- 1 file changed, 120 insertions(+), 70 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index 238260ea3..a909add0b 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "time" "github.com/fleetdm/fleet/v4/ee/server/calendar" @@ -13,6 +14,7 @@ import ( "github.com/go-kit/log" kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" + "golang.org/x/sync/errgroup" ) func newCalendarSchedule( @@ -57,7 +59,6 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L return nil } googleCalendarIntegrationConfig := appConfig.Integrations.GoogleCalendar[0] - calendar := createUserCalendarFromConfig(ctx, googleCalendarIntegrationConfig, logger) domain := googleCalendarIntegrationConfig.Domain teams, err := ds.ListTeams(ctx, fleet.TeamFilter{ @@ -71,7 +72,7 @@ func cronCalendarEvents(ctx context.Context, ds fleet.Datastore, logger kitlog.L for _, team := range teams { if err := cronCalendarEventsForTeam( - ctx, ds, calendar, *team, appConfig.OrgInfo.OrgName, domain, logger, + ctx, ds, googleCalendarIntegrationConfig, *team, appConfig.OrgInfo.OrgName, domain, logger, ); err != nil { level.Info(logger).Log("msg", "events calendar cron", "team_id", team.ID, "err", err) } @@ -92,7 +93,7 @@ func createUserCalendarFromConfig(ctx context.Context, config *fleet.GoogleCalen func cronCalendarEventsForTeam( ctx context.Context, ds fleet.Datastore, - calendar fleet.UserCalendar, + calendarConfig *fleet.GoogleCalendarIntegration, team fleet.Team, orgName string, domain string, @@ -161,13 +162,13 @@ func cronCalendarEventsForTeam( // We execute this first to remove any calendar events for a user that is now passing // policies on one of its hosts, and possibly create a new calendar event if they have // another failing host on the same team. - if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendar, passingHosts); err != nil { + if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendarConfig, passingHosts, logger); err != nil { level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err) } // Process hosts that are failing calendar policies. if err := processCalendarFailingHosts( - ctx, ds, calendar, orgName, failingHosts, logger, + ctx, ds, calendarConfig, orgName, failingHosts, logger, ); err != nil { level.Info(logger).Log("msg", "processing failing hosts", "err", err) } @@ -185,67 +186,82 @@ func cronCalendarEventsForTeam( func processCalendarFailingHosts( ctx context.Context, ds fleet.Datastore, - userCalendar fleet.UserCalendar, + calendarConfig *fleet.GoogleCalendarIntegration, orgName string, hosts []fleet.HostPolicyMembershipData, logger kitlog.Logger, ) error { hosts = filterHostsWithSameEmail(hosts) - for _, host := range hosts { - logger := log.With(logger, "host_id", host.HostID) + const consumers = 100 + hostsCh := make(chan fleet.HostPolicyMembershipData) + g, ctx := errgroup.WithContext(ctx) - hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email) + for i := 0; i < consumers; i++ { + g.Go(func() error { + for host := range hostsCh { + logger := log.With(logger, "host_id", host.HostID) - expiredEvent := false - if err == nil { - if hostCalendarEvent.HostID != host.HostID { - // This calendar event belongs to another host with this associated email, - // thus we skip this entry. - continue // continue with next host - } - if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusPending { - // This can happen if the host went offline (and never returned results) - // after setting the webhook as pending. - continue // continue with next host - } - now := time.Now() - webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent - if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) { - // If the webhook already fired today and the policies are still failing - // we give a grace period of one day for the host before we schedule a new event. - continue // continue with next host - } - if calendarEvent.EndTime.Before(now) { - expiredEvent = true - } - } + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email) - if err := userCalendar.Configure(host.Email); err != nil { - return fmt.Errorf("configure user calendar: %w", err) - } + expiredEvent := false + if err == nil { + if hostCalendarEvent.HostID != host.HostID { + // This calendar event belongs to another host with this associated email, + // thus we skip this entry. + continue // continue with next host + } + if hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusPending { + // This can happen if the host went offline (and never returned results) + // after setting the webhook as pending. + continue // continue with next host + } + now := time.Now() + webhookAlreadyFired := hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusSent + if webhookAlreadyFired && sameDate(now, calendarEvent.StartTime) { + // If the webhook already fired today and the policies are still failing + // we give a grace period of one day for the host before we schedule a new event. + continue // continue with next host + } + if calendarEvent.EndTime.Before(now) { + expiredEvent = true + } + } - switch { - case err == nil && !expiredEvent: - if err := processFailingHostExistingCalendarEvent( - ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host, - ); err != nil { - level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err) - continue // continue with next host + userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) + if err := userCalendar.Configure(host.Email); err != nil { + return fmt.Errorf("configure user calendar: %w", err) + } + + switch { + case err == nil && !expiredEvent: + if err := processFailingHostExistingCalendarEvent( + ctx, ds, userCalendar, orgName, hostCalendarEvent, calendarEvent, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host existing calendar event", "err", err) + continue // continue with next host + } + case fleet.IsNotFound(err) || expiredEvent: + if err := processFailingHostCreateCalendarEvent( + ctx, ds, userCalendar, orgName, host, + ); err != nil { + level.Info(logger).Log("msg", "process failing host create calendar event", "err", err) + continue // continue with next host + } + default: + return fmt.Errorf("get calendar event: %w", err) + } } - case fleet.IsNotFound(err) || expiredEvent: - if err := processFailingHostCreateCalendarEvent( - ctx, ds, userCalendar, orgName, host, - ); err != nil { - level.Info(logger).Log("msg", "process failing host create calendar event", "err", err) - continue // continue with next host - } - default: - return fmt.Errorf("get calendar event: %w", err) - } + return nil + }) } - return nil + for _, host := range hosts { + hostsCh <- host + } + close(hostsCh) + + return g.Wait() } func filterHostsWithSameEmail(hosts []fleet.HostPolicyMembershipData) []fleet.HostPolicyMembershipData { @@ -437,27 +453,61 @@ func addBusinessDay(date time.Time) time.Time { func removeCalendarEventsFromPassingHosts( ctx context.Context, ds fleet.Datastore, - userCalendar fleet.UserCalendar, + calendarConfig *fleet.GoogleCalendarIntegration, hosts []fleet.HostPolicyMembershipData, + logger kitlog.Logger, ) error { + hostIDsByEmail := make(map[string][]uint) for _, host := range hosts { - hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, host.Email) - switch { - case err == nil: - if hostCalendarEvent.HostID != host.HostID { - // This calendar event belongs to another host, thus we skip this entry. - continue - } - case fleet.IsNotFound(err): - continue - default: - return fmt.Errorf("get calendar event from DB: %w", err) - } - if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { - return fmt.Errorf("delete user calendar event: %w", err) - } + hostIDsByEmail[host.Email] = append(hostIDsByEmail[host.Email], host.HostID) } - return nil + type emailWithHosts struct { + email string + hostIDs []uint + } + emails := make([]emailWithHosts, 0, len(hostIDsByEmail)) + for email, hostIDs := range hostIDsByEmail { + emails = append(emails, emailWithHosts{ + email: email, + hostIDs: hostIDs, + }) + } + + const consumers = 100 + emailsCh := make(chan emailWithHosts) + g, ctx := errgroup.WithContext(ctx) + + for i := 0; i < consumers; i++ { + g.Go(func() error { + for email := range emailsCh { + + hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, email.email) + switch { + case err == nil: + if ok := slices.Contains(email.hostIDs, hostCalendarEvent.HostID); !ok { + // None of the hosts belong to this calendar event. + continue + } + case fleet.IsNotFound(err): + continue + default: + return fmt.Errorf("get calendar event from DB: %w", err) + } + userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) + if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { + return fmt.Errorf("delete user calendar event: %w", err) + } + } + return nil + }) + } + + for _, emailWithHostIDs := range emails { + emailsCh <- emailWithHostIDs + } + close(emailsCh) + + return g.Wait() } func logHostsWithoutAssociatedEmail( From 35a21d5f0c8339c9d98bd2b5efa9f2a2028f3c2f Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 22 Mar 2024 14:29:05 -0500 Subject: [PATCH 24/36] Calendar helper scripts for testing (#17798) Calendar helper scripts for testing --- tools/calendar/README.md | 6 + tools/calendar/delete-events/delete-events.go | 74 ++++++++++++ tools/calendar/move-events/move-events.go | 111 ++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 tools/calendar/delete-events/delete-events.go create mode 100644 tools/calendar/move-events/move-events.go diff --git a/tools/calendar/README.md b/tools/calendar/README.md index 5d222e45d..bab2f481b 100644 --- a/tools/calendar/README.md +++ b/tools/calendar/README.md @@ -1,3 +1,9 @@ +# Helper methods for Google calendar + +To delete all downtime events from a Google Calendar, use `delete-events/delete-events.go` + +To move all downtime events from multiple Google Calendars to a specific time, use `move-events/move-events.go` + # Calendar server for load testing Test calendar server that provides a REST API for managing events. diff --git a/tools/calendar/delete-events/delete-events.go b/tools/calendar/delete-events/delete-events.go new file mode 100644 index 000000000..cfa0b2ce0 --- /dev/null +++ b/tools/calendar/delete-events/delete-events.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "flag" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/option" + "log" + "os" +) + +// Delete all events with eventTitle from the primary calendar of the user. + +var ( + serviceEmail = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL") + privateKey = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY") +) + +const ( + eventTitle = "💻🚫Downtime" +) + +func main() { + if serviceEmail == "" || privateKey == "" { + log.Fatal("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL and FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY must be set") + } + userEmail := flag.String("user", "", "User email to impersonate") + flag.Parse() + if *userEmail == "" { + log.Fatal("--user is required") + } + + ctx := context.Background() + conf := &jwt.Config{ + Email: serviceEmail, + Scopes: []string{ + "https://www.googleapis.com/auth/calendar.events", "https://www.googleapis.com/auth/calendar.settings.readonly", + }, + PrivateKey: []byte(privateKey), + TokenURL: google.JWTTokenURL, + Subject: *userEmail, + } + client := conf.Client(ctx) + // Create a new calendar service + service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + log.Fatalf("Unable to create Calendar service: %v", err) + } + numberDeleted := 0 + for { + list, err := service.Events.List("primary").EventTypes("default").MaxResults(1000).OrderBy("startTime").SingleEvents(true).ShowDeleted(false).Q(eventTitle).Do() + if err != nil { + log.Fatalf("Unable to retrieve list of events: %v", err) + } + if len(list.Items) == 0 { + break + } + for _, item := range list.Items { + if item.Summary == eventTitle { + err = service.Events.Delete("primary", item.Id).Do() + if err != nil { + log.Fatalf("Unable to delete event: %v", err) + } + numberDeleted++ + if numberDeleted%10 == 0 { + log.Printf("Deleted %d events", numberDeleted) + } + } + } + } + log.Printf("DONE. Deleted %d events total", numberDeleted) +} diff --git a/tools/calendar/move-events/move-events.go b/tools/calendar/move-events/move-events.go new file mode 100644 index 000000000..2e66b56e6 --- /dev/null +++ b/tools/calendar/move-events/move-events.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "flag" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/option" + "log" + "os" + "strings" + "sync" + "time" +) + +// Move all events with eventTitle from the primary calendar of the user to the new time. +// Only events in the future relative to the new event time are moved. In other words, if the current event time is in the past, it is not moved. + +var ( + serviceEmail = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL") + privateKey = os.Getenv("FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY") +) + +const ( + eventTitle = "💻🚫Downtime" +) + +func main() { + if serviceEmail == "" || privateKey == "" { + log.Fatal("FLEET_TEST_GOOGLE_CALENDAR_SERVICE_EMAIL and FLEET_TEST_GOOGLE_CALENDAR_PRIVATE_KEY must be set") + } + userEmails := flag.String("users", "", "Comma-separated list of user emails to impersonate") + dateTimeStr := flag.String("datetime", "", "Event time in "+time.RFC3339+" format") + flag.Parse() + if *userEmails == "" { + log.Fatal("--users are required") + } + if *dateTimeStr == "" { + log.Fatal("--datetime is required") + } + dateTime, err := time.Parse(time.RFC3339, *dateTimeStr) + if err != nil { + log.Fatalf("Unable to parse datetime: %v", err) + } + dateTimeEndStr := dateTime.Add(30 * time.Minute).Format(time.RFC3339) + userEmailList := strings.Split(*userEmails, ",") + if len(userEmailList) == 0 { + log.Fatal("No user emails provided") + } + + ctx := context.Background() + + var wg sync.WaitGroup + + for _, userEmail := range userEmailList { + wg.Add(1) + go func(userEmail string) { + defer wg.Done() + conf := &jwt.Config{ + Email: serviceEmail, + Scopes: []string{ + "https://www.googleapis.com/auth/calendar.events", "https://www.googleapis.com/auth/calendar.settings.readonly", + }, + PrivateKey: []byte(privateKey), + TokenURL: google.JWTTokenURL, + Subject: userEmail, + } + client := conf.Client(ctx) + // Create a new calendar service + service, err := calendar.NewService(ctx, option.WithHTTPClient(client)) + if err != nil { + log.Fatalf("Unable to create Calendar service: %v", err) + } + + numberMoved := 0 + for { + list, err := service.Events.List("primary").EventTypes("default"). + MaxResults(1000). + OrderBy("startTime"). + SingleEvents(true). + ShowDeleted(false). + TimeMin(dateTimeEndStr). + Q(eventTitle). + Do() + if err != nil { + log.Fatalf("Unable to retrieve list of events: %v", err) + } + if len(list.Items) == 0 { + break + } + for _, item := range list.Items { + if item.Summary == eventTitle { + item.Start.DateTime = dateTime.Format(time.RFC3339) + item.End.DateTime = dateTime.Add(30 * time.Minute).Format(time.RFC3339) + _, err := service.Events.Update("primary", item.Id, item).Do() + if err != nil { + log.Fatalf("Unable to update event: %v", err) + } + numberMoved++ + } + } + } + log.Printf("Moved %d events for %s", numberMoved, userEmail) + }(userEmail) + } + + // Wait for all goroutines to finish + wg.Wait() + +} From 2e5656328013c591484a91624c51823b46fb98d7 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Fri, 22 Mar 2024 15:53:51 -0500 Subject: [PATCH 25/36] Adding retry logic when rate limited by Google Calendar API. (#17810) Adding retry logic when rate limited by Google Calendar API. --- cmd/fleet/calendar_cron.go | 4 +- ee/server/calendar/google_calendar.go | 87 ++++++++++++++++++++++----- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index a909add0b..2813a0160 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -193,7 +193,7 @@ func processCalendarFailingHosts( ) error { hosts = filterHostsWithSameEmail(hosts) - const consumers = 100 + const consumers = 20 hostsCh := make(chan fleet.HostPolicyMembershipData) g, ctx := errgroup.WithContext(ctx) @@ -473,7 +473,7 @@ func removeCalendarEventsFromPassingHosts( }) } - const consumers = 100 + const consumers = 20 emailsCh := make(chan emailWithHosts) g, ctx := errgroup.WithContext(ctx) diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index 42f8b7b0d..ea7376b16 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -10,6 +10,7 @@ import ( "regexp" "time" + "github.com/cenkalti/backoff/v4" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" kitlog "github.com/go-kit/log" @@ -72,7 +73,7 @@ func NewGoogleCalendar(config *GoogleCalendarConfig) *GoogleCalendar { case config.IntegrationConfig.ApiKey[fleet.GoogleCalendarEmail] == mockEmail: config.API = &GoogleCalendarMockAPI{config.Logger} default: - config.API = &GoogleCalendarLowLevelAPI{} + config.API = &GoogleCalendarLowLevelAPI{logger: config.Logger} } return &GoogleCalendar{ config: config, @@ -95,6 +96,7 @@ type eventDetails struct { type GoogleCalendarLowLevelAPI struct { service *calendar.Service + logger kitlog.Logger } // Configure creates a new Google Calendar service using the provided credentials. @@ -126,31 +128,77 @@ func adjustEmail(email string) string { } func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetSetting(name string) (*calendar.Setting, error) { - return lowLevelAPI.service.Settings.Get(name).Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + return lowLevelAPI.service.Settings.Get(name).Do() + }, + ) + return result.(*calendar.Setting), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { - return lowLevelAPI.service.Events.Insert(calendarID, event).Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + return lowLevelAPI.service.Events.Insert(calendarID, event).Do() + }, + ) + return result.(*calendar.Event), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) GetEvent(id, eTag string) (*calendar.Event, error) { - return lowLevelAPI.service.Events.Get(calendarID, id).IfNoneMatch(eTag).Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + return lowLevelAPI.service.Events.Get(calendarID, id).IfNoneMatch(eTag).Do() + }, + ) + return result.(*calendar.Event), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) ListEvents(timeMin, timeMax string) (*calendar.Events, error) { - // Default maximum number of events returned is 250, which should be sufficient for most calendars. - return lowLevelAPI.service.Events.List(calendarID). - EventTypes("default"). - OrderBy("startTime"). - SingleEvents(true). - TimeMin(timeMin). - TimeMax(timeMax). - ShowDeleted(false). - Do() + result, err := lowLevelAPI.withRetry( + func() (any, error) { + // Default maximum number of events returned is 250, which should be sufficient for most calendars. + return lowLevelAPI.service.Events.List(calendarID). + EventTypes("default"). + OrderBy("startTime"). + SingleEvents(true). + TimeMin(timeMin). + TimeMax(timeMax). + ShowDeleted(false). + Do() + }, + ) + return result.(*calendar.Events), err } func (lowLevelAPI *GoogleCalendarLowLevelAPI) DeleteEvent(id string) error { - return lowLevelAPI.service.Events.Delete(calendarID, id).Do() + _, err := lowLevelAPI.withRetry( + func() (any, error) { + return nil, lowLevelAPI.service.Events.Delete(calendarID, id).Do() + }, + ) + return err +} + +func (lowLevelAPI *GoogleCalendarLowLevelAPI) withRetry(fn func() (any, error)) (any, error) { + retryStrategy := backoff.NewExponentialBackOff() + retryStrategy.MaxElapsedTime = 10 * time.Minute + var result any + err := backoff.Retry( + func() error { + var err error + result, err = fn() + if err != nil { + if isRateLimited(err) { + level.Debug(lowLevelAPI.logger).Log("msg", "rate limited by Google calendar API", "err", err) + return err + } + return backoff.Permanent(err) + } + return nil + }, retryStrategy, + ) + return result, err } func (c *GoogleCalendar) Configure(userEmail string) error { @@ -308,6 +356,17 @@ func isAlreadyDeleted(err error) bool { return ok && ae.Code == http.StatusGone } +func isRateLimited(err error) bool { + if err == nil { + return false + } + var ae *googleapi.Error + ok := errors.As(err, &ae) + return ok && (ae.Code == http.StatusTooManyRequests || + (ae.Code == http.StatusForbidden && + (ae.Message == "Rate Limit Exceeded" || ae.Message == "User Rate Limit Exceeded" || ae.Message == "Calendar usage limits exceeded."))) +} + func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) { var details eventDetails err := json.Unmarshal(event.Data, &details) From 9090d8541f5192f49bebd859f3fc35409081d1c9 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Mon, 25 Mar 2024 07:21:56 -0300 Subject: [PATCH 26/36] Calendar update event if meeting occurring now (#17815) #17441 --------- Co-authored-by: Victor Lyuboslavsky --- cmd/fleet/calendar_cron.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index 2813a0160..c6113d718 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -296,9 +296,7 @@ func processFailingHostExistingCalendarEvent( updated := false now := time.Now() - // Check the user calendar every 30 minutes (and not every time) - // to reduce load on both Fleet and the calendar service. - if time.Since(calendarEvent.UpdatedAt) > 30*time.Minute { + if shouldReloadCalendarEvent(now, calendarEvent, hostCalendarEvent) { var err error updatedEvent, _, err = calendar.GetAndUpdateEvent(calendarEvent, func(conflict bool) string { return generateCalendarEventBody(orgName, host.HostDisplayName, conflict) @@ -365,6 +363,24 @@ func processFailingHostExistingCalendarEvent( return nil } +func shouldReloadCalendarEvent(now time.Time, calendarEvent *fleet.CalendarEvent, hostCalendarEvent *fleet.HostCalendarEvent) bool { + // Check the user calendar every 30 minutes (and not every cron run) + // to reduce load on both Fleet and the calendar service. + if time.Since(calendarEvent.UpdatedAt) > 30*time.Minute { + return true + } + // If the event is supposed to be happening now, we want to check if the user moved/deleted the + // event on the last minute. + if eventHappeningNow(now, calendarEvent) && hostCalendarEvent.WebhookStatus == fleet.CalendarWebhookStatusNone { + return true + } + return false +} + +func eventHappeningNow(now time.Time, calendarEvent *fleet.CalendarEvent) bool { + return !now.Before(calendarEvent.StartTime) && now.Before(calendarEvent.EndTime) +} + func sameDate(t1 time.Time, t2 time.Time) bool { y1, m1, d1 := t1.Date() y2, m2, d2 := t2.Date() From 51cd71f46423d1e9595ce940696e954dfbf4c268 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Mon, 25 Mar 2024 15:15:13 -0300 Subject: [PATCH 27/36] Fix concurrency bug in calendar cron (#17832) #17441 --- cmd/fleet/calendar_cron.go | 60 +-- cmd/fleet/calendar_cron_test.go | 453 ++++++++++++++++++++- ee/server/calendar/google_calendar_mock.go | 26 +- 3 files changed, 504 insertions(+), 35 deletions(-) diff --git a/cmd/fleet/calendar_cron.go b/cmd/fleet/calendar_cron.go index c6113d718..e4b1927d5 100644 --- a/cmd/fleet/calendar_cron.go +++ b/cmd/fleet/calendar_cron.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "slices" + "sync" "time" "github.com/fleetdm/fleet/v4/ee/server/calendar" @@ -14,7 +15,6 @@ import ( "github.com/go-kit/log" kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" - "golang.org/x/sync/errgroup" ) func newCalendarSchedule( @@ -162,16 +162,18 @@ func cronCalendarEventsForTeam( // We execute this first to remove any calendar events for a user that is now passing // policies on one of its hosts, and possibly create a new calendar event if they have // another failing host on the same team. - if err := removeCalendarEventsFromPassingHosts(ctx, ds, calendarConfig, passingHosts, logger); err != nil { - level.Info(logger).Log("msg", "removing calendar events from passing hosts", "err", err) - } + start := time.Now() + removeCalendarEventsFromPassingHosts(ctx, ds, calendarConfig, passingHosts, logger) + level.Debug(logger).Log( + "msg", "passing_hosts", "took", time.Since(start), + ) // Process hosts that are failing calendar policies. - if err := processCalendarFailingHosts( - ctx, ds, calendarConfig, orgName, failingHosts, logger, - ); err != nil { - level.Info(logger).Log("msg", "processing failing hosts", "err", err) - } + start = time.Now() + processCalendarFailingHosts(ctx, ds, calendarConfig, orgName, failingHosts, logger) + level.Debug(logger).Log( + "msg", "failing_hosts", "took", time.Since(start), + ) // At last we want to log the hosts that are failing and don't have an associated email. logHostsWithoutAssociatedEmail( @@ -190,15 +192,18 @@ func processCalendarFailingHosts( orgName string, hosts []fleet.HostPolicyMembershipData, logger kitlog.Logger, -) error { +) { hosts = filterHostsWithSameEmail(hosts) const consumers = 20 hostsCh := make(chan fleet.HostPolicyMembershipData) - g, ctx := errgroup.WithContext(ctx) + var wg sync.WaitGroup for i := 0; i < consumers; i++ { - g.Go(func() error { + wg.Add(+1) + go func() { + defer wg.Done() + for host := range hostsCh { logger := log.With(logger, "host_id", host.HostID) @@ -230,7 +235,8 @@ func processCalendarFailingHosts( userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) if err := userCalendar.Configure(host.Email); err != nil { - return fmt.Errorf("configure user calendar: %w", err) + level.Error(logger).Log("msg", "configure user calendar", "err", err) + continue // continue with next host } switch { @@ -249,11 +255,11 @@ func processCalendarFailingHosts( continue // continue with next host } default: - return fmt.Errorf("get calendar event: %w", err) + level.Error(logger).Log("msg", "get calendar event from db", "err", err) + continue // continue with next host } } - return nil - }) + }() } for _, host := range hosts { @@ -261,7 +267,7 @@ func processCalendarFailingHosts( } close(hostsCh) - return g.Wait() + wg.Wait() } func filterHostsWithSameEmail(hosts []fleet.HostPolicyMembershipData) []fleet.HostPolicyMembershipData { @@ -472,7 +478,7 @@ func removeCalendarEventsFromPassingHosts( calendarConfig *fleet.GoogleCalendarIntegration, hosts []fleet.HostPolicyMembershipData, logger kitlog.Logger, -) error { +) { hostIDsByEmail := make(map[string][]uint) for _, host := range hosts { hostIDsByEmail[host.Email] = append(hostIDsByEmail[host.Email], host.HostID) @@ -491,10 +497,13 @@ func removeCalendarEventsFromPassingHosts( const consumers = 20 emailsCh := make(chan emailWithHosts) - g, ctx := errgroup.WithContext(ctx) + var wg sync.WaitGroup for i := 0; i < consumers; i++ { - g.Go(func() error { + wg.Add(+1) + go func() { + defer wg.Done() + for email := range emailsCh { hostCalendarEvent, calendarEvent, err := ds.GetHostCalendarEventByEmail(ctx, email.email) @@ -507,15 +516,16 @@ func removeCalendarEventsFromPassingHosts( case fleet.IsNotFound(err): continue default: - return fmt.Errorf("get calendar event from DB: %w", err) + level.Error(logger).Log("msg", "get calendar event from DB", "err", err) + continue } userCalendar := createUserCalendarFromConfig(ctx, calendarConfig, logger) if err := deleteCalendarEvent(ctx, ds, userCalendar, calendarEvent); err != nil { - return fmt.Errorf("delete user calendar event: %w", err) + level.Error(logger).Log("msg", "delete user calendar event", "err", err) + continue } } - return nil - }) + }() } for _, emailWithHostIDs := range emails { @@ -523,7 +533,7 @@ func removeCalendarEventsFromPassingHosts( } close(emailsCh) - return g.Wait() + wg.Wait() } func logHostsWithoutAssociatedEmail( diff --git a/cmd/fleet/calendar_cron_test.go b/cmd/fleet/calendar_cron_test.go index c6db483c5..4d9133377 100644 --- a/cmd/fleet/calendar_cron_test.go +++ b/cmd/fleet/calendar_cron_test.go @@ -2,12 +2,21 @@ package main import ( "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/ee/server/calendar" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" kitlog "github.com/go-kit/log" - "os" - "testing" - "time" "github.com/stretchr/testify/require" ) @@ -188,5 +197,441 @@ func TestEventForDifferentHost(t *testing.T) { err := cronCalendarEvents(ctx, ds, logger) require.NoError(t, err) - +} + +func TestCalendarEventsMultipleHosts(t *testing.T) { + ds := new(mock.Store) + ctx := context.Background() + logger := kitlog.With(kitlog.NewLogfmtLogger(os.Stdout)) + t.Cleanup(func() { + calendar.ClearMockEvents() + }) + + // TODO(lucas): Test! + webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + requestBodyBytes, err := io.ReadAll(r.Body) + require.NoError(t, err) + t.Logf("webhook request: %s\n", requestBodyBytes) + })) + t.Cleanup(func() { + webhookServer.Close() + }) + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Domain: "example.com", + ApiKey: map[string]string{ + fleet.GoogleCalendarEmail: "calendar-mock@example.com", + }, + }, + }, + }, + }, nil + } + + teamID1 := uint(1) + ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { + return []*fleet.Team{ + { + ID: teamID1, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + }, nil + } + + policyID1 := uint(10) + policyID2 := uint(11) + ds.GetCalendarPoliciesFunc = func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + require.Equal(t, teamID1, teamID) + return []fleet.PolicyCalendarData{ + { + ID: policyID1, + Name: "Policy 1", + }, + { + ID: policyID2, + Name: "Policy 2", + }, + }, nil + } + + hostID1, userEmail1 := uint(100), "user1@example.com" + hostID2, userEmail2 := uint(101), "user2@example.com" + hostID3, userEmail3 := uint(102), "user3@other.com" + hostID4, userEmail4 := uint(103), "user4@other.com" + + ds.GetTeamHostsPolicyMembershipsFunc = func( + ctx context.Context, domain string, teamID uint, policyIDs []uint, + ) ([]fleet.HostPolicyMembershipData, error) { + require.Equal(t, teamID1, teamID) + require.Equal(t, []uint{policyID1, policyID2}, policyIDs) + return []fleet.HostPolicyMembershipData{ + { + HostID: hostID1, + Email: userEmail1, + Passing: false, + }, + { + HostID: hostID2, + Email: userEmail2, + Passing: true, + }, + { + HostID: hostID3, + Email: userEmail3, + Passing: false, + }, + { + HostID: hostID4, + Email: userEmail4, + Passing: true, + }, + }, nil + } + + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + return nil, nil, notFoundErr{} + } + + ds.CreateOrUpdateCalendarEventFunc = func(ctx context.Context, + email string, + startTime, endTime time.Time, + data []byte, + hostID uint, + webhookStatus fleet.CalendarWebhookStatus, + ) (*fleet.CalendarEvent, error) { + switch email { + case userEmail1: + require.Equal(t, hostID1, hostID) + case userEmail2: + require.Equal(t, hostID2, hostID) + case userEmail3: + require.Equal(t, hostID3, hostID) + case userEmail4: + require.Equal(t, hostID4, hostID) + } + require.Equal(t, fleet.CalendarWebhookStatusNone, webhookStatus) + require.NotEmpty(t, data) + require.NotZero(t, startTime) + require.NotZero(t, endTime) + // Currently, the returned calendar event is unused. + return nil, nil + } + + err := cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) +} + +type notFoundErr struct{} + +func (n notFoundErr) IsNotFound() bool { + return true +} + +func (n notFoundErr) Error() string { + return "not found" +} + +func TestCalendarEvents1KHosts(t *testing.T) { + ds := new(mock.Store) + ctx := context.Background() + var logger kitlog.Logger + if os.Getenv("CALENDAR_TEST_LOGGING") != "" { + logger = kitlog.With(kitlog.NewLogfmtLogger(os.Stdout)) + } else { + logger = kitlog.NewNopLogger() + } + t.Cleanup(func() { + calendar.ClearMockEvents() + }) + + // TODO(lucas): Use for the test. + webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "POST", r.Method) + requestBodyBytes, err := io.ReadAll(r.Body) + require.NoError(t, err) + t.Logf("webhook request: %s\n", requestBodyBytes) + })) + t.Cleanup(func() { + webhookServer.Close() + }) + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ + Integrations: fleet.Integrations{ + GoogleCalendar: []*fleet.GoogleCalendarIntegration{ + { + Domain: "example.com", + ApiKey: map[string]string{ + fleet.GoogleCalendarEmail: "calendar-mock@example.com", + }, + }, + }, + }, + }, nil + } + + teamID1 := uint(1) + teamID2 := uint(2) + teamID3 := uint(3) + teamID4 := uint(4) + teamID5 := uint(5) + ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) { + return []*fleet.Team{ + { + ID: teamID1, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID2, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID3, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID4, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + { + ID: teamID5, + Config: fleet.TeamConfig{ + Integrations: fleet.TeamIntegrations{ + GoogleCalendar: &fleet.TeamGoogleCalendarIntegration{ + Enable: true, + WebhookURL: webhookServer.URL, + }, + }, + }, + }, + }, nil + } + + policyID1 := uint(10) + policyID2 := uint(11) + policyID3 := uint(12) + policyID4 := uint(13) + policyID5 := uint(14) + policyID6 := uint(15) + policyID7 := uint(16) + policyID8 := uint(17) + policyID9 := uint(18) + policyID10 := uint(19) + ds.GetCalendarPoliciesFunc = func(ctx context.Context, teamID uint) ([]fleet.PolicyCalendarData, error) { + switch teamID { + case teamID1: + return []fleet.PolicyCalendarData{ + { + ID: policyID1, + Name: "Policy 1", + }, + { + ID: policyID2, + Name: "Policy 2", + }, + }, nil + case teamID2: + return []fleet.PolicyCalendarData{ + { + ID: policyID3, + Name: "Policy 3", + }, + { + ID: policyID4, + Name: "Policy 4", + }, + }, nil + case teamID3: + return []fleet.PolicyCalendarData{ + { + ID: policyID5, + Name: "Policy 5", + }, + { + ID: policyID6, + Name: "Policy 6", + }, + }, nil + case teamID4: + return []fleet.PolicyCalendarData{ + { + ID: policyID7, + Name: "Policy 7", + }, + { + ID: policyID8, + Name: "Policy 8", + }, + }, nil + case teamID5: + return []fleet.PolicyCalendarData{ + { + ID: policyID9, + Name: "Policy 9", + }, + { + ID: policyID10, + Name: "Policy 10", + }, + }, nil + default: + return nil, notFoundErr{} + } + } + + hosts := make([]fleet.HostPolicyMembershipData, 0, 1000) + for i := 0; i < 1000; i++ { + hosts = append(hosts, fleet.HostPolicyMembershipData{ + Email: fmt.Sprintf("user%d@example.com", i), + Passing: i%2 == 0, + HostID: uint(i), + HostDisplayName: fmt.Sprintf("display_name%d", i), + HostHardwareSerial: fmt.Sprintf("serial%d", i), + }) + } + + ds.GetTeamHostsPolicyMembershipsFunc = func( + ctx context.Context, domain string, teamID uint, policyIDs []uint, + ) ([]fleet.HostPolicyMembershipData, error) { + var start, end int + switch teamID { + case teamID1: + start, end = 0, 200 + case teamID2: + start, end = 200, 400 + case teamID3: + start, end = 400, 600 + case teamID4: + start, end = 600, 800 + case teamID5: + start, end = 800, 1000 + } + return hosts[start:end], nil + } + + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + return nil, nil, notFoundErr{} + } + + eventsCreated := 0 + var eventsCreatedMu sync.Mutex + + eventPerHost := make(map[uint]*fleet.CalendarEvent) + + ds.CreateOrUpdateCalendarEventFunc = func(ctx context.Context, + email string, + startTime, endTime time.Time, + data []byte, + hostID uint, + webhookStatus fleet.CalendarWebhookStatus, + ) (*fleet.CalendarEvent, error) { + require.Equal(t, fmt.Sprintf("user%d@example.com", hostID), email) + eventsCreatedMu.Lock() + eventsCreated += 1 + eventPerHost[hostID] = &fleet.CalendarEvent{ + ID: hostID, + Email: email, + StartTime: startTime, + EndTime: endTime, + Data: data, + UpdateCreateTimestamps: fleet.UpdateCreateTimestamps{ + CreateTimestamp: fleet.CreateTimestamp{ + CreatedAt: time.Now(), + }, + UpdateTimestamp: fleet.UpdateTimestamp{ + UpdatedAt: time.Now(), + }, + }, + } + eventsCreatedMu.Unlock() + require.Equal(t, fleet.CalendarWebhookStatusNone, webhookStatus) + require.NotEmpty(t, data) + require.NotZero(t, startTime) + require.NotZero(t, endTime) + // Currently, the returned calendar event is unused. + return nil, nil + } + + err := cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) + + createdCalendarEvents := calendar.ListGoogleMockEvents() + require.Equal(t, eventsCreated, 500) + require.Len(t, createdCalendarEvents, 500) + + hosts = make([]fleet.HostPolicyMembershipData, 0, 1000) + for i := 0; i < 1000; i++ { + hosts = append(hosts, fleet.HostPolicyMembershipData{ + Email: fmt.Sprintf("user%d@example.com", i), + Passing: true, + HostID: uint(i), + HostDisplayName: fmt.Sprintf("display_name%d", i), + HostHardwareSerial: fmt.Sprintf("serial%d", i), + }) + } + + ds.GetHostCalendarEventByEmailFunc = func(ctx context.Context, email string) (*fleet.HostCalendarEvent, *fleet.CalendarEvent, error) { + hostID, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(email, "user"), "@example.com")) + require.NoError(t, err) + if hostID%2 == 0 { + return nil, nil, notFoundErr{} + } + require.Contains(t, eventPerHost, uint(hostID)) + return &fleet.HostCalendarEvent{ + ID: uint(hostID), + HostID: uint(hostID), + CalendarEventID: uint(hostID), + WebhookStatus: fleet.CalendarWebhookStatusNone, + }, eventPerHost[uint(hostID)], nil + } + + ds.DeleteCalendarEventFunc = func(ctx context.Context, calendarEventID uint) error { + return nil + } + + err = cronCalendarEvents(ctx, ds, logger) + require.NoError(t, err) + + createdCalendarEvents = calendar.ListGoogleMockEvents() + require.Len(t, createdCalendarEvents, 0) } diff --git a/ee/server/calendar/google_calendar_mock.go b/ee/server/calendar/google_calendar_mock.go index 255f8d87c..08e3a72e2 100644 --- a/ee/server/calendar/google_calendar_mock.go +++ b/ee/server/calendar/google_calendar_mock.go @@ -3,23 +3,26 @@ package calendar import ( "context" "errors" - kitlog "github.com/go-kit/log" - "google.golang.org/api/calendar/v3" - "google.golang.org/api/googleapi" "net/http" "os" "strconv" "sync" "time" + + kitlog "github.com/go-kit/log" + "google.golang.org/api/calendar/v3" + "google.golang.org/api/googleapi" ) type GoogleCalendarMockAPI struct { logger kitlog.Logger } -var mockEvents = make(map[string]*calendar.Event) -var mu sync.Mutex -var id uint64 +var ( + mockEvents = make(map[string]*calendar.Event) + mu sync.Mutex + id uint64 +) const latency = 500 * time.Millisecond @@ -44,6 +47,7 @@ func (lowLevelAPI *GoogleCalendarMockAPI) GetSetting(name string) (*calendar.Set } func (lowLevelAPI *GoogleCalendarMockAPI) CreateEvent(event *calendar.Event) (*calendar.Event, error) { + time.Sleep(latency) mu.Lock() defer mu.Unlock() id += 1 @@ -79,3 +83,13 @@ func (lowLevelAPI *GoogleCalendarMockAPI) DeleteEvent(id string) error { delete(mockEvents, id) return nil } + +func ListGoogleMockEvents() map[string]*calendar.Event { + return mockEvents +} + +func ClearMockEvents() { + mu.Lock() + defer mu.Unlock() + mockEvents = make(map[string]*calendar.Event) +} From 72662291b0c39e99373ca911d5a570542f79c0c4 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 25 Mar 2024 16:11:46 -0500 Subject: [PATCH 28/36] Adding embedded timezone database. --- server/fleet/calendar.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/fleet/calendar.go b/server/fleet/calendar.go index 592fff430..5eb4597f4 100644 --- a/server/fleet/calendar.go +++ b/server/fleet/calendar.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "time" + _ "time/tzdata" // embed timezone information in the program "github.com/fleetdm/fleet/v4/server" ) From b92733b0e3eba5af1a79fd65ae0ef7b998f1fea4 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 25 Mar 2024 16:59:02 -0500 Subject: [PATCH 29/36] Adding another error message for rate limiting. --- ee/server/calendar/google_calendar.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ee/server/calendar/google_calendar.go b/ee/server/calendar/google_calendar.go index ea7376b16..7283269df 100644 --- a/ee/server/calendar/google_calendar.go +++ b/ee/server/calendar/google_calendar.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "regexp" + "strings" "time" "github.com/cenkalti/backoff/v4" @@ -364,7 +365,7 @@ func isRateLimited(err error) bool { ok := errors.As(err, &ae) return ok && (ae.Code == http.StatusTooManyRequests || (ae.Code == http.StatusForbidden && - (ae.Message == "Rate Limit Exceeded" || ae.Message == "User Rate Limit Exceeded" || ae.Message == "Calendar usage limits exceeded."))) + (ae.Message == "Rate Limit Exceeded" || ae.Message == "User Rate Limit Exceeded" || ae.Message == "Calendar usage limits exceeded." || strings.HasPrefix(ae.Message, "Quota exceeded")))) } func (c *GoogleCalendar) unmarshalDetails(event *fleet.CalendarEvent) (*eventDetails, error) { From 9861f6eca633b0f7577a7af96b090a988e31c050 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Tue, 26 Mar 2024 13:47:20 -0500 Subject: [PATCH 30/36] Added Fleet in your calendar changes file. --- changes/17230-fleet-in-your-calendar | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/17230-fleet-in-your-calendar diff --git a/changes/17230-fleet-in-your-calendar b/changes/17230-fleet-in-your-calendar new file mode 100644 index 000000000..299239a07 --- /dev/null +++ b/changes/17230-fleet-in-your-calendar @@ -0,0 +1,5 @@ +Added integration with Google Calendar. +- Fleet admins can enable Google Calendar integration by using a Google service account with domain-wide delegation. +- Calendar integration is enabled at the team level for specific team policies. +- If the policy is failing, a calendar event will be put on the host user's calendar for the 3rd Tuesday of the month. +- During the event, Fleet will fire a webhook. IT admins should use this webhook to trigger a script or MDM command that will remediate the issue. From 0b04e7ea9fc6521ed25f5a7f5f3c56aeeaa0cf68 Mon Sep 17 00:00:00 2001 From: Dante Catalfamo <43040593+dantecatalfamo@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:33:52 -0400 Subject: [PATCH 31/36] Allow EnrollmentState to be in status '3' for MDM clients (#17868) #17692 Recently there was a change that filtered out hosts in `EnrollmentState` 3. This change may cause some hosts that are in otherwise good health to appear unresponsive to MDM in the management UI. This change will allow hosts with `EnrollmentStatus` 3 show as enrolled. The root cause of some hosts being in state 3 is still not entirely clear, but may have to do with either trying to re-enroll once already enrolled, or windows updates causing some sort of issue with fleet. Despite the "failed" `EnrollmentState` 3, the host will still display that the system is managed by Fleet, and will actively sync. --- changes/17692-enrollment-state-3.md | 1 + server/service/osquery_utils/queries.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/17692-enrollment-state-3.md diff --git a/changes/17692-enrollment-state-3.md b/changes/17692-enrollment-state-3.md new file mode 100644 index 000000000..5703a31fd --- /dev/null +++ b/changes/17692-enrollment-state-3.md @@ -0,0 +1 @@ +- Fix a bug where valid MDM enrollments would show up as unmanaged (EnrollmentState 3) diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index e65124e4e..b7b3d6409 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -506,7 +506,7 @@ var extraDetailQueries = map[string]DetailQuery{ -- coalesce to 'unknown' and keep that state in the list -- in order to account for hosts that might not have this -- key, and servers - WHERE COALESCE(e.state, '0') IN ('0', '1', '2') + WHERE COALESCE(e.state, '0') IN ('0', '1', '2', '3') LIMIT 1; `, DirectIngestFunc: directIngestMDMWindows, From 593a59255c4f39af33d8d60e9f2b68206b4ae1b4 Mon Sep 17 00:00:00 2001 From: Dave Herder <27025660+dherder@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:55:06 -0700 Subject: [PATCH 32/36] Update workstations-canary.yml (#17761) - Add script to install Bitdefender in canary workstations --- it-and-security/lib/windows-install-bitdefender.ps1 | 8 ++++++++ it-and-security/teams/workstations-canary.yml | 1 + 2 files changed, 9 insertions(+) create mode 100644 it-and-security/lib/windows-install-bitdefender.ps1 diff --git a/it-and-security/lib/windows-install-bitdefender.ps1 b/it-and-security/lib/windows-install-bitdefender.ps1 new file mode 100644 index 000000000..17fe5495a --- /dev/null +++ b/it-and-security/lib/windows-install-bitdefender.ps1 @@ -0,0 +1,8 @@ +$ResolveWingetPath = Resolve-Path "C:\Program Files\WindowsApps\Microsoft.DesktopAppInstaller_*_x64__8wekyb3d8bbwe" + if ($ResolveWingetPath){ + $WingetPath = $ResolveWingetPath[-1].Path + } + +$config +Set-Location $wingetpath +.\winget.exe install --id=Bitdefender.Bitdefender -e -h --accept-package-agreements --accept-source-agreements --disable-interactivity diff --git a/it-and-security/teams/workstations-canary.yml b/it-and-security/teams/workstations-canary.yml index 2ceefcce9..d2c42b0cd 100644 --- a/it-and-security/teams/workstations-canary.yml +++ b/it-and-security/teams/workstations-canary.yml @@ -53,6 +53,7 @@ controls: - path: ../lib/macos-remove-old-nudge.sh - path: ../lib/windows-remove-fleetd.ps1 - path: ../lib/windows-turn-off-mdm.ps1 + - path: ../lib/windows-install-bitdefender.ps1 policies: - path: ../lib/macos-device-health.policies.yml - path: ../lib/windows-device-health.policies.yml From 0f74ae0109926245bd0e2f86324873f8126f9cb7 Mon Sep 17 00:00:00 2001 From: Jacob Shandling <61553566+jacobshandling@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:15:26 -0700 Subject: [PATCH 33/36] =?UTF-8?q?UI=20=E2=80=93=20Handle=20missing=20/=20`?= =?UTF-8?q?null`=20`smtp=5Fsettings`=20(#17850)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Addresses #17065 Settings cleanly renders as empty and disabled despite nonexistent `smtp_settings` from config response: ![Screenshot 2024-03-25 at 3 52 05 PM](https://github.com/fleetdm/fleet/assets/61553566/85c2a9af-7cc2-48b7-9ecf-604496813204) - [x] Changes file added for user-visible changes in `changes/` - [x] Manual QA for all new/changed functionality --------- Co-authored-by: Jacob Shandling --- changes/17065-null-smtp_settings | 1 + frontend/interfaces/config.ts | 2 +- frontend/pages/AccountPage/AccountPage.tsx | 6 +++- .../cards/Advanced/Advanced.tsx | 24 ++++++++-------- .../admin/OrgSettingsPage/cards/Smtp/Smtp.tsx | 28 +++++++++---------- .../UsersPage/UsersPage.tsx | 5 +++- .../components/UsersTable/UsersTable.tsx | 14 +++++++--- 7 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 changes/17065-null-smtp_settings diff --git a/changes/17065-null-smtp_settings b/changes/17065-null-smtp_settings new file mode 100644 index 000000000..b37de2555 --- /dev/null +++ b/changes/17065-null-smtp_settings @@ -0,0 +1 @@ +- Fix a bug where `null` or excluded `smtp_settings` caused a UI 500. diff --git a/frontend/interfaces/config.ts b/frontend/interfaces/config.ts index 8eee167f1..42f0e4469 100644 --- a/frontend/interfaces/config.ts +++ b/frontend/interfaces/config.ts @@ -122,7 +122,7 @@ export interface IConfig { }; sandbox_enabled: boolean; server_settings: IConfigServerSettings; - smtp_settings: { + smtp_settings?: { enable_smtp: boolean; configured: boolean; sender_address: string; diff --git a/frontend/pages/AccountPage/AccountPage.tsx b/frontend/pages/AccountPage/AccountPage.tsx index 02cfcb360..2042a41ec 100644 --- a/frontend/pages/AccountPage/AccountPage.tsx +++ b/frontend/pages/AccountPage/AccountPage.tsx @@ -91,7 +91,11 @@ const AccountPage = ({ router }: IAccountPageProps): JSX.Element | null => { await usersAPI.update(currentUser.id, updated); let accountUpdatedFlashMessage = "Account updated"; if (updated.email) { - accountUpdatedFlashMessage += `: A confirmation email was sent from ${config?.smtp_settings.sender_address} to ${updated.email}`; + // always present, this for typing + const senderAddressMessage = config?.smtp_settings?.sender_address + ? ` from ${config?.smtp_settings?.sender_address}` + : ""; + accountUpdatedFlashMessage += `: A confirmation email was sent${senderAddressMessage} to ${updated.email}`; setPendingEmail(updated.email); } diff --git a/frontend/pages/admin/OrgSettingsPage/cards/Advanced/Advanced.tsx b/frontend/pages/admin/OrgSettingsPage/cards/Advanced/Advanced.tsx index 0ff0936a3..bd0704ce4 100644 --- a/frontend/pages/admin/OrgSettingsPage/cards/Advanced/Advanced.tsx +++ b/frontend/pages/admin/OrgSettingsPage/cards/Advanced/Advanced.tsx @@ -20,9 +20,9 @@ const Advanced = ({ isUpdatingSettings, }: IAppConfigFormProps): JSX.Element => { const [formData, setFormData] = useState({ - domain: appConfig.smtp_settings.domain || "", - verifySSLCerts: appConfig.smtp_settings.verify_ssl_certs || false, - enableStartTLS: appConfig.smtp_settings.enable_start_tls, + domain: appConfig.smtp_settings?.domain || "", + verifySSLCerts: appConfig.smtp_settings?.verify_ssl_certs || false, + enableStartTLS: appConfig.smtp_settings?.enable_start_tls, enableHostExpiry: appConfig.host_expiry_settings.host_expiry_enabled || false, hostExpiryWindow: appConfig.host_expiry_settings.host_expiry_window || 0, @@ -74,16 +74,16 @@ const Advanced = ({ scripts_disabled: disableScripts, }, smtp_settings: { - enable_smtp: appConfig.smtp_settings.enable_smtp || false, - sender_address: appConfig.smtp_settings.sender_address || "", - server: appConfig.smtp_settings.server || "", - port: Number(appConfig.smtp_settings.port), - authentication_type: appConfig.smtp_settings.authentication_type || "", - user_name: appConfig.smtp_settings.user_name || "", - password: appConfig.smtp_settings.password || "", - enable_ssl_tls: appConfig.smtp_settings.enable_ssl_tls || false, + enable_smtp: appConfig.smtp_settings?.enable_smtp || false, + sender_address: appConfig.smtp_settings?.sender_address || "", + server: appConfig.smtp_settings?.server || "", + port: Number(appConfig.smtp_settings?.port), + authentication_type: appConfig.smtp_settings?.authentication_type || "", + user_name: appConfig.smtp_settings?.user_name || "", + password: appConfig.smtp_settings?.password || "", + enable_ssl_tls: appConfig.smtp_settings?.enable_ssl_tls || false, authentication_method: - appConfig.smtp_settings.authentication_method || "", + appConfig.smtp_settings?.authentication_method || "", domain, verify_ssl_certs: verifySSLCerts, enable_start_tls: enableStartTLS, diff --git a/frontend/pages/admin/OrgSettingsPage/cards/Smtp/Smtp.tsx b/frontend/pages/admin/OrgSettingsPage/cards/Smtp/Smtp.tsx index cb4b0d980..45af19339 100644 --- a/frontend/pages/admin/OrgSettingsPage/cards/Smtp/Smtp.tsx +++ b/frontend/pages/admin/OrgSettingsPage/cards/Smtp/Smtp.tsx @@ -31,16 +31,16 @@ const Smtp = ({ const { isPremiumTier } = useContext(AppContext); const [formData, setFormData] = useState({ - enableSMTP: appConfig.smtp_settings.enable_smtp || false, - smtpSenderAddress: appConfig.smtp_settings.sender_address || "", - smtpServer: appConfig.smtp_settings.server || "", - smtpPort: appConfig.smtp_settings.port, - smtpEnableSSLTLS: appConfig.smtp_settings.enable_ssl_tls || false, - smtpAuthenticationType: appConfig.smtp_settings.authentication_type || "", - smtpUsername: appConfig.smtp_settings.user_name || "", - smtpPassword: appConfig.smtp_settings.password || "", + enableSMTP: appConfig.smtp_settings?.enable_smtp || false, + smtpSenderAddress: appConfig.smtp_settings?.sender_address || "", + smtpServer: appConfig.smtp_settings?.server || "", + smtpPort: appConfig.smtp_settings?.port, + smtpEnableSSLTLS: appConfig.smtp_settings?.enable_ssl_tls || false, + smtpAuthenticationType: appConfig.smtp_settings?.authentication_type || "", + smtpUsername: appConfig.smtp_settings?.user_name || "", + smtpPassword: appConfig.smtp_settings?.password || "", smtpAuthenticationMethod: - appConfig.smtp_settings.authentication_method || "", + appConfig.smtp_settings?.authentication_method || "", }); const { @@ -116,9 +116,9 @@ const Smtp = ({ password: smtpPassword, enable_ssl_tls: smtpEnableSSLTLS, authentication_method: smtpAuthenticationMethod, - domain: appConfig.smtp_settings.domain || "", - verify_ssl_certs: appConfig.smtp_settings.verify_ssl_certs || false, - enable_start_tls: appConfig.smtp_settings.enable_start_tls, + domain: appConfig.smtp_settings?.domain || "", + verify_ssl_certs: appConfig.smtp_settings?.verify_ssl_certs || false, + enable_start_tls: appConfig.smtp_settings?.enable_start_tls, }, }; @@ -282,13 +282,13 @@ const Smtp = ({ !sesConfigured ? ( - {appConfig.smtp_settings.configured + {appConfig.smtp_settings?.configured ? "CONFIGURED" : "NOT CONFIGURED"} diff --git a/frontend/pages/admin/TeamManagementPage/TeamDetailsWrapper/UsersPage/UsersPage.tsx b/frontend/pages/admin/TeamManagementPage/TeamDetailsWrapper/UsersPage/UsersPage.tsx index 1e56bcfb0..55d2d7cb3 100644 --- a/frontend/pages/admin/TeamManagementPage/TeamDetailsWrapper/UsersPage/UsersPage.tsx +++ b/frontend/pages/admin/TeamManagementPage/TeamDetailsWrapper/UsersPage/UsersPage.tsx @@ -217,9 +217,12 @@ const UsersPage = ({ location, router }: ITeamSubnavProps): JSX.Element => { inviteAPI .create(requestData) .then(() => { + const senderAddressMessage = config?.smtp_settings?.sender_address + ? ` from ${config?.smtp_settings?.sender_address}` + : ""; renderFlash( "success", - `An invitation email was sent from ${config?.smtp_settings.sender_address} to ${formData.email}.` + `An invitation email was sent${senderAddressMessage} to ${formData.email}.` ); refetchUsers(); toggleCreateUserModal(); diff --git a/frontend/pages/admin/UserManagementPage/components/UsersTable/UsersTable.tsx b/frontend/pages/admin/UserManagementPage/components/UsersTable/UsersTable.tsx index 02955ae2a..76e01bf91 100644 --- a/frontend/pages/admin/UserManagementPage/components/UsersTable/UsersTable.tsx +++ b/frontend/pages/admin/UserManagementPage/components/UsersTable/UsersTable.tsx @@ -225,9 +225,12 @@ const UsersTable = ({ router }: IUsersTableProps): JSX.Element => { invitesAPI .create(requestData) .then(() => { + const senderAddressMessage = config?.smtp_settings?.sender_address + ? ` from ${config?.smtp_settings?.sender_address}` + : ""; renderFlash( "success", - `An invitation email was sent from ${config?.smtp_settings.sender_address} to ${formData.email}.` + `An invitation email was sent${senderAddressMessage} to ${formData.email}.` ); toggleCreateUserModal(); refetchInvites(); @@ -302,7 +305,10 @@ const UsersTable = ({ router }: IUsersTableProps): JSX.Element => { let userUpdatedFlashMessage = `Successfully edited ${formData.name}`; if (userData?.email !== formData.email) { - userUpdatedFlashMessage += `: A confirmation email was sent from ${config?.smtp_settings.sender_address} to ${formData.email}`; + const senderAddressMessage = config?.smtp_settings?.sender_address + ? ` from ${config?.smtp_settings?.sender_address}` + : ""; + userUpdatedFlashMessage += `: A confirmation email was sent${senderAddressMessage} to ${formData.email}`; } const userUpdatedEmailError = "A user with this email address already exists"; @@ -463,7 +469,7 @@ const UsersTable = ({ router }: IUsersTableProps): JSX.Element => { onSubmit={onEditUser} availableTeams={teams || []} isPremiumTier={isPremiumTier || false} - smtpConfigured={config?.smtp_settings.configured || false} + smtpConfigured={config?.smtp_settings?.configured || false} sesConfigured={config?.email?.backend === "ses" || false} canUseSso={config?.sso_settings.enable_sso || false} isSsoEnabled={userData?.sso_enabled} @@ -486,7 +492,7 @@ const UsersTable = ({ router }: IUsersTableProps): JSX.Element => { defaultGlobalRole="observer" defaultTeams={[]} isPremiumTier={isPremiumTier || false} - smtpConfigured={config?.smtp_settings.configured || false} + smtpConfigured={config?.smtp_settings?.configured || false} sesConfigured={config?.email?.backend === "ses" || false} canUseSso={config?.sso_settings.enable_sso || false} isUpdatingUsers={isUpdatingUsers} From 37c1c3c8fd3aea00a6b493042b8296d6dba8ee69 Mon Sep 17 00:00:00 2001 From: Sam Pfluger <108141731+Sampfluger88@users.noreply.github.com> Date: Tue, 26 Mar 2024 21:55:24 -0500 Subject: [PATCH 34/36] Removing broken image from bizops page. (#17877) --- handbook/business-operations/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/handbook/business-operations/README.md b/handbook/business-operations/README.md index 4c8f16f44..491da3468 100644 --- a/handbook/business-operations/README.md +++ b/handbook/business-operations/README.md @@ -283,7 +283,6 @@ You can confirm that the device has been ordered correctly by following these st - Use the device serial number to find the device. - Note: if the device cannot be found, you will need to manually enroll the device. - View device settings and ensure the "MDM Server" selected is "Fleet Dogfood". -Screenshot 2023-11-21 at 11 08 50 AM On occasion there will be a need to manually enroll a macOS host in dogfood. This could be due to a BYOD arrangement, or because the Fleetie getting the device is in a country when DEP (automatic enrollment) isn't supported. To manually enroll a macOS host in dogfood, follow these steps: - If you have physical access to the macOS host, use Apple Configurator (docs are [here](https://support.apple.com/guide/apple-business-manager/add-devices-from-apple-configurator-axm200a54d59/web)). From d28810a8f2038d1c42151717727b33796827c394 Mon Sep 17 00:00:00 2001 From: Sam Pfluger <108141731+Sampfluger88@users.noreply.github.com> Date: Wed, 27 Mar 2024 03:07:29 -0500 Subject: [PATCH 35/36] Add link to "Why don't we sell like everyone else?" (#17878) --- handbook/company/why-this-way.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handbook/company/why-this-way.md b/handbook/company/why-this-way.md index e03cabb07..d363e5e01 100644 --- a/handbook/company/why-this-way.md +++ b/handbook/company/why-this-way.md @@ -247,7 +247,7 @@ For example, here is the [philosophy behind Fleet's bug report template](https:/ ## Why don't we sell like everyone else? -Many companies encourage salespeople to "spray and pray" email blasts, and to do whatever it takes to close deals. This can sometimes be temporarily effective. But Fleet takes a [🟠longer-term](https://fleetdm.com/handbook/company#ownership) approach: +Many companies encourage salespeople to ["spray and pray"](https://www.linkedin.com/posts/amstech_the-rampant-abuse-of-linkedin-connections-activity-7178412289413246978-Ci0I?utm_source=share&utm_medium=member_ios) email blasts, and to do whatever it takes to close deals. This can sometimes be temporarily effective. But Fleet takes a [🟠longer-term](https://fleetdm.com/handbook/company#ownership) approach: - **No spam.** Fleet is deliberate and thoughtful in the way we do outreach, whether that's for community-building, education, or [🧊 conversation-starting](https://github.com/fleetdm/confidential/blob/main/cold-outbound-strategy.md). - **Be a helper.** We focus on [🔴being helpers](https://fleetdm.com/handbook/company#empathy). Always be depositing value. This is how we create a virtuous cycle. (That doesn't mean sharing a random article; it means genuinely hearing, doing whatever it takes to fully understand, and offering only advice or links that we would actually want.) We are genuinely curious and desperate to help, because creating real value for people is the way we win. - **Engineers first.** We always talk to engineers first, and learn how it's going. Security and IT engineers are the people closest to the work, and the people best positioned to know what their organizations need. From 1261956a89bd13dc03df5bc5477656b9b45ffc1f Mon Sep 17 00:00:00 2001 From: Sam Pfluger <108141731+Sampfluger88@users.noreply.github.com> Date: Wed, 27 Mar 2024 03:12:53 -0500 Subject: [PATCH 36/36] =?UTF-8?q?Add=20link=20to=20"Why=20not=20continuous?= =?UTF-8?q?ly=20generate=20REST=20API=20reference=20docs=20fr=E2=80=A6=20(?= =?UTF-8?q?#17879)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handbook/company/why-this-way.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handbook/company/why-this-way.md b/handbook/company/why-this-way.md index d363e5e01..010680a7b 100644 --- a/handbook/company/why-this-way.md +++ b/handbook/company/why-this-way.md @@ -131,7 +131,7 @@ Besides the exceptions above, Fleet does not use any other repositories. Other ## Why not continuously generate REST API reference docs from javadoc-style code comments? Here are a few of the drawbacks that we have experienced when generating docs via tools like Swagger or OpenAPI, and some of the advantages of doing it by hand with Markdown. -- Markdown gives us more control over how the docs are compiled, what annotations we can include, and how we present the information to the end-user. +- Markdown gives us more control over how the docs are compiled, what annotations we can include, and how we [present the information to the end-user](https://x.com/wesleytodd/status/1769810305448616185?s=46&t=4_cwTxqV5IXDLBvCm8KI6Q). - Markdown is more accessible. Anyone can edit Fleet's docs directly from our website without needing coding experience. - A single Markdown file reduces the amount of surface area to manage that comes from spreading code comments across multiple files throughout the codebase. (see ["Why do we use one repo?"](#why-do-we-use-one-repo)). - Autogenerated docs can become just as outdated as handmade docs, except since they are siloed, they require more skills to edit.