fix redis concurrency issue in health metrics + remove ongoing status count (#10717)

### Context
For calendar and message sync job health monitoring, we used to
increment a counter in redis cache which could lead to concurrency
issue.

### Solution
- Update to a set structure in place of counter + use sAdd redis method
which is atomic
- Each minute another counter was incremented on a new cache key ->
Update to a 15s window
- Remove ONGOING status not needed. We only need status at job end (or
fail).


###  Potential improvements
- Check for cache key existence before fetching data to avoid useless
call to redis ?

closes https://github.com/twentyhq/twenty/issues/10070

packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts

@@ -6,6 +6,7 @@ import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decora
 import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
 import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
 import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
+import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type';
 import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
 import {
   CalendarChannelSyncStage,
@@ -79,11 +80,6 @@ export class CalendarChannelSyncStatusService {
       syncStatus: CalendarChannelSyncStatus.ONGOING,
       syncStageStartedAt: new Date().toISOString(),
     });
-
-    await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
-      CalendarChannelSyncStatus.ONGOING,
-      calendarChannelIds.length,
-    );
   }
 
   public async resetAndScheduleFullCalendarEventListFetch(
@@ -183,9 +179,10 @@ export class CalendarChannelSyncStatusService {
 
     await this.schedulePartialCalendarEventListFetch(calendarChannelIds);
 
-    await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
+    await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
+      HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
       CalendarChannelSyncStatus.ACTIVE,
-      calendarChannelIds.length,
+      calendarChannelIds,
     );
   }
 
@@ -213,9 +210,10 @@ export class CalendarChannelSyncStatusService {
       syncStage: CalendarChannelSyncStage.FAILED,
     });
 
-    await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
+    await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
+      HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
       CalendarChannelSyncStatus.FAILED_UNKNOWN,
-      calendarChannelIds.length,
+      calendarChannelIds,
     );
   }
 
@@ -268,9 +266,10 @@ export class CalendarChannelSyncStatusService {
       workspaceId,
     );
 
-    await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
+    await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
+      HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
       CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
-      calendarChannelIds.length,
+      calendarChannelIds,
     );
   }
 

packages/twenty-server/src/modules/messaging/common/services/message-channel-sync-status.service.ts

@@ -6,6 +6,7 @@ import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decora
 import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
 import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
 import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
+import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type';
 import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
 import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
 import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
@@ -129,11 +130,6 @@ export class MessageChannelSyncStatusService {
       syncStatus: MessageChannelSyncStatus.ONGOING,
       syncStageStartedAt: new Date().toISOString(),
     });
-
-    await this.healthCacheService.incrementMessageChannelSyncJobByStatusCounter(
-      MessageChannelSyncStatus.ONGOING,
-      messageChannelIds.length,
-    );
   }
 
   public async markAsCompletedAndSchedulePartialMessageListFetch(
@@ -156,9 +152,10 @@ export class MessageChannelSyncStatusService {
       syncedAt: new Date().toISOString(),
     });
 
-    await this.healthCacheService.incrementMessageChannelSyncJobByStatusCounter(
+    await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
+      HealthCounterCacheKeys.MessageChannelSyncJobByStatus,
       MessageChannelSyncStatus.ACTIVE,
-      messageChannelIds.length,
+      messageChannelIds,
     );
   }
 
@@ -202,9 +199,10 @@ export class MessageChannelSyncStatusService {
       syncStatus: MessageChannelSyncStatus.FAILED_UNKNOWN,
     });
 
-    await this.healthCacheService.incrementMessageChannelSyncJobByStatusCounter(
+    await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
+      HealthCounterCacheKeys.MessageChannelSyncJobByStatus,
       MessageChannelSyncStatus.FAILED_UNKNOWN,
-      messageChannelIds.length,
+      messageChannelIds,
     );
   }
 
@@ -232,9 +230,10 @@ export class MessageChannelSyncStatusService {
       syncStatus: MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
     });
 
-    await this.healthCacheService.incrementMessageChannelSyncJobByStatusCounter(
+    await this.healthCacheService.updateMessageOrCalendarChannelSyncJobByStatusCache(
+      HealthCounterCacheKeys.MessageChannelSyncJobByStatus,
       MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
-      messageChannelIds.length,
+      messageChannelIds,
     );
 
     const connectedAccountRepository =