Fix cron module structure (#4933)

This PR introduces a new folder structure for business modules.
Cron commands and jobs are now stored within the same module/folder at
the root of the business module
e.g: /modules/messaging/crons/commands instead of
/modules/messaging/commands/crons
Patterns are now inside their own cron-command files since they don't
need to be exported
Ideally cronJobs and cronCommands should have their logic within the
same class but it's a bit harder than expected due to how commanderjs
and our worker need both some class heritage check, hence the first
approach is to move them in the same folder

Also Messaging fullsync/partialsync V2 has been dropped since this is
the only used version => Breaking change for ongoing jobs and crons.
Jobs can be dropped but we will need to re-run our crons (only
cron:messaging:gmail-fetch-messages-from-cache)
This commit is contained in:
Weiko
2024-04-12 14:43:03 +02:00
committed by GitHub
parent a6b38d76ce
commit f4fda221b7
18 changed files with 136 additions and 122 deletions
@@ -1,21 +1,23 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { DatabaseCommandModule } from 'src/database/commands/database-command.module'; import { DatabaseCommandModule } from 'src/database/commands/database-command.module';
import { MessagingCommandModule } from 'src/modules/messaging/commands/messaging-command.module';
import { WorkspaceHealthCommandModule } from 'src/engine/workspace-manager/workspace-health/commands/workspace-health-command.module'; import { WorkspaceHealthCommandModule } from 'src/engine/workspace-manager/workspace-health/commands/workspace-health-command.module';
import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module'; import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-cleaner/workspace-cleaner.module';
import { WorkspaceCalendarSyncCommandsModule } from 'src/modules/calendar/commands/workspace-calendar-sync-commands.module'; import { CalendarCronCommandsModule } from 'src/modules/calendar/crons/commands/calendar-cron-commands.module';
import { AppModule } from 'src/app.module'; import { AppModule } from 'src/app.module';
import { WorkspaceMigrationRunnerCommandsModule } from 'src/engine/workspace-manager/workspace-migration-runner/commands/workspace-sync-metadata-commands.module'; import { WorkspaceMigrationRunnerCommandsModule } from 'src/engine/workspace-manager/workspace-migration-runner/commands/workspace-sync-metadata-commands.module';
import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module';
import { MessagingCronCommandsModule } from 'src/modules/messaging/crons/commands/messaging-cron-commands.module';
import { CalendarCommandsModule } from 'src/modules/calendar/commands/calendar-commands.module';
@Module({ @Module({
imports: [ imports: [
AppModule, AppModule,
WorkspaceSyncMetadataCommandsModule, WorkspaceSyncMetadataCommandsModule,
DatabaseCommandModule, DatabaseCommandModule,
MessagingCommandModule, MessagingCronCommandsModule,
WorkspaceCalendarSyncCommandsModule, CalendarCronCommandsModule,
CalendarCommandsModule,
WorkspaceCleanerModule, WorkspaceCleanerModule,
WorkspaceHealthCommandModule, WorkspaceHealthCommandModule,
WorkspaceMigrationRunnerCommandsModule, WorkspaceMigrationRunnerCommandsModule,
@@ -1,43 +1,43 @@
import { Inject, Injectable } from '@nestjs/common'; import { Injectable, Inject } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import { InjectRepository } from '@nestjs/typeorm';
import { Repository, EntityManager } from 'typeorm';
import { v4 } from 'uuid'; import { v4 } from 'uuid';
import { EntityManager, Repository } from 'typeorm';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import {
GoogleCalendarSyncJob,
GoogleCalendarSyncJobData,
} from 'src/modules/calendar/jobs/google-calendar-sync.job';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { import {
FeatureFlagEntity, FeatureFlagEntity,
FeatureFlagKeys, FeatureFlagKeys,
} from 'src/engine/core-modules/feature-flag/feature-flag.entity'; } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
GmailFullSyncV2Job, import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
GmailFullSyncV2JobData, import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import {
GoogleCalendarSyncJobData,
GoogleCalendarSyncJob,
} from 'src/modules/calendar/jobs/google-calendar-sync.job';
import { CalendarChannelRepository } from 'src/modules/calendar/repositories/calendar-channel.repository';
import {
CalendarChannelObjectMetadata,
CalendarChannelVisibility,
} from 'src/modules/calendar/standard-objects/calendar-channel.object-metadata';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { import {
ConnectedAccountObjectMetadata, ConnectedAccountObjectMetadata,
ConnectedAccountProvider, ConnectedAccountProvider,
} from 'src/modules/connected-account/standard-objects/connected-account.object-metadata'; } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository'; import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
import { import {
MessageChannelObjectMetadata, MessageChannelObjectMetadata,
MessageChannelType, MessageChannelType,
MessageChannelVisibility, MessageChannelVisibility,
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata'; } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
import { import {
CalendarChannelObjectMetadata, GmailFullSyncJobData,
CalendarChannelVisibility, GmailFullSyncJob,
} from 'src/modules/calendar/standard-objects/calendar-channel.object-metadata'; } from 'src/modules/messaging/jobs/gmail-full-sync.job';
import { CalendarChannelRepository } from 'src/modules/calendar/repositories/calendar-channel.repository';
@Injectable() @Injectable()
export class GoogleAPIsService { export class GoogleAPIsService {
@@ -171,8 +171,8 @@ export class GoogleAPIsService {
isCalendarEnabled: boolean, isCalendarEnabled: boolean,
) { ) {
if (this.environmentService.get('MESSAGING_PROVIDER_GMAIL_ENABLED')) { if (this.environmentService.get('MESSAGING_PROVIDER_GMAIL_ENABLED')) {
await this.messageQueueService.add<GmailFullSyncV2JobData>( await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncV2Job.name, GmailFullSyncJob.name,
{ {
workspaceId, workspaceId,
connectedAccountId, connectedAccountId,
@@ -1,58 +1,58 @@
import { HttpModule } from '@nestjs/axios';
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core'; import { ModuleRef } from '@nestjs/core';
import { HttpModule } from '@nestjs/axios';
import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeOrmModule } from '@nestjs/typeorm';
import { CallWebhookJobsJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job';
import { CallWebhookJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { CleanInactiveWorkspaceJob } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-inactive-workspace.job';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { EmailSenderJob } from 'src/engine/integrations/email/email-sender.job';
import { UserModule } from 'src/engine/core-modules/user/user.module';
import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
import { MatchParticipantJob } from 'src/modules/connected-account/jobs/match-participant.job';
import { GmailPartialSyncCronJob } from 'src/modules/messaging/jobs/crons/gmail-partial-sync.cron.job';
import { MessagingCreateCompanyAndContactAfterSyncJob } from 'src/modules/messaging/jobs/messaging-create-company-and-contact-after-sync.job';
import { AutoCompaniesAndContactsCreationModule } from 'src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module';
import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module'; import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module';
import { DataSeedDemoWorkspaceJob } from 'src/database/commands/data-seed-demo-workspace/jobs/data-seed-demo-workspace.job'; import { DataSeedDemoWorkspaceJob } from 'src/database/commands/data-seed-demo-workspace/jobs/data-seed-demo-workspace.job';
import { DeleteConnectedAccountAssociatedMessagingDataJob } from 'src/modules/messaging/jobs/delete-connected-account-associated-messaging-data.job'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { ThreadCleanerModule } from 'src/modules/messaging/services/thread-cleaner/thread-cleaner.module'; import { CallWebhookJobsJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job';
import { UpdateSubscriptionJob } from 'src/engine/core-modules/billing/jobs/update-subscription.job'; import { CallWebhookJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';
import { StripeModule } from 'src/engine/core-modules/billing/stripe/stripe.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { GoogleCalendarSyncJob } from 'src/modules/calendar/jobs/google-calendar-sync.job';
import { CalendarEventCleanerModule } from 'src/modules/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
import { RecordPositionBackfillJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/record-position-backfill.job'; import { RecordPositionBackfillJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/record-position-backfill.job';
import { RecordPositionBackfillModule } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-module';
import { GoogleCalendarSyncModule } from 'src/modules/calendar/services/google-calendar-sync.module';
import { GoogleAPIRefreshAccessTokenModule } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.module';
import { MessageParticipantModule } from 'src/modules/messaging/services/message-participant/message-participant.module';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job'; import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job'; import { RecordPositionBackfillModule } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-module';
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata'; import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { UpdateSubscriptionJob } from 'src/engine/core-modules/billing/jobs/update-subscription.job';
import { StripeModule } from 'src/engine/core-modules/billing/stripe/stripe.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';
import { UserModule } from 'src/engine/core-modules/user/user.module';
import { HandleWorkspaceMemberDeletedJob } from 'src/engine/core-modules/workspace/handle-workspace-member-deleted.job'; import { HandleWorkspaceMemberDeletedJob } from 'src/engine/core-modules/workspace/handle-workspace-member-deleted.job';
import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module'; import { EmailSenderJob } from 'src/engine/integrations/email/email-sender.job';
import { GmailFullSyncV2Job } from 'src/modules/messaging/jobs/gmail-full-sync-v2.job'; import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
import { GmailPartialSyncV2Job } from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job'; import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { GoogleCalendarSyncCronJob } from 'src/modules/calendar/jobs/crons/google-calendar-sync.cron.job'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module'; import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { UnmatchParticipantJob } from 'src/modules/connected-account/jobs/unmatch-participant.job'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CleanInactiveWorkspaceJob } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-inactive-workspace.job';
import { GoogleCalendarSyncCronJob } from 'src/modules/calendar/crons/jobs/google-calendar-sync.cron.job';
import { CalendarCreateCompanyAndContactAfterSyncJob } from 'src/modules/calendar/jobs/calendar-create-company-and-contact-after-sync.job'; import { CalendarCreateCompanyAndContactAfterSyncJob } from 'src/modules/calendar/jobs/calendar-create-company-and-contact-after-sync.job';
import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/modules/calendar/jobs/delete-connected-account-associated-calendar-data.job'; import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/modules/calendar/jobs/delete-connected-account-associated-calendar-data.job';
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/jobs/crons/fetch-all-messages-from-cache.cron.job'; import { GoogleCalendarSyncJob } from 'src/modules/calendar/jobs/google-calendar-sync.job';
import { CalendarEventCleanerModule } from 'src/modules/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module';
import { GoogleCalendarSyncModule } from 'src/modules/calendar/services/google-calendar-sync.module';
import { AutoCompaniesAndContactsCreationModule } from 'src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module';
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
import { MatchParticipantJob } from 'src/modules/connected-account/jobs/match-participant.job';
import { UnmatchParticipantJob } from 'src/modules/connected-account/jobs/unmatch-participant.job';
import { GoogleAPIRefreshAccessTokenModule } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.module';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
import { GmailFetchMessagesFromCacheCronJob } from 'src/modules/messaging/crons/jobs/gmail-fetch-messages-from-cache.cron.job';
import { GmailPartialSyncCronJob } from 'src/modules/messaging/crons/jobs/gmail-partial-sync.cron.job';
import { DeleteConnectedAccountAssociatedMessagingDataJob } from 'src/modules/messaging/jobs/delete-connected-account-associated-messaging-data.job';
import { GmailFullSyncJob } from 'src/modules/messaging/jobs/gmail-full-sync.job';
import { GmailPartialSyncJob } from 'src/modules/messaging/jobs/gmail-partial-sync.job';
import { MessagingCreateCompanyAndContactAfterSyncJob } from 'src/modules/messaging/jobs/messaging-create-company-and-contact-after-sync.job';
import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module';
import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module';
import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module';
import { MessageParticipantModule } from 'src/modules/messaging/services/message-participant/message-participant.module';
import { ThreadCleanerModule } from 'src/modules/messaging/services/thread-cleaner/thread-cleaner.module';
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
@Module({ @Module({
imports: [ imports: [
@@ -155,16 +155,16 @@ import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/jobs/cro
useClass: SaveEventToDbJob, useClass: SaveEventToDbJob,
}, },
{ {
provide: FetchAllMessagesFromCacheCronJob.name, provide: GmailFetchMessagesFromCacheCronJob.name,
useClass: FetchAllMessagesFromCacheCronJob, useClass: GmailFetchMessagesFromCacheCronJob,
}, },
{ {
provide: GmailFullSyncV2Job.name, provide: GmailFullSyncJob.name,
useClass: GmailFullSyncV2Job, useClass: GmailFullSyncJob,
}, },
{ {
provide: GmailPartialSyncV2Job.name, provide: GmailPartialSyncJob.name,
useClass: GmailPartialSyncV2Job, useClass: GmailPartialSyncJob,
}, },
{ {
provide: GoogleCalendarSyncCronJob.name, provide: GoogleCalendarSyncCronJob.name,
@@ -1,14 +1,17 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { GoogleCalendarSyncCommand } from 'src/modules/calendar/commands/google-calendar-sync.command';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module'; import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { GoogleCalendarSyncCommand } from 'src/modules/calendar/commands/google-calendar-sync.command';
import { CalendarChannelObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel.object-metadata';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata'; import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { StartGoogleCalendarSyncCronJobCommand } from 'src/modules/calendar/commands/start-google-calendar-sync.cron.command';
@Module({ @Module({
imports: [ imports: [
ObjectMetadataRepositoryModule.forFeature([ConnectedAccountObjectMetadata]), ObjectMetadataRepositoryModule.forFeature([
ConnectedAccountObjectMetadata,
CalendarChannelObjectMetadata,
]),
], ],
providers: [GoogleCalendarSyncCommand, StartGoogleCalendarSyncCronJobCommand], providers: [GoogleCalendarSyncCommand],
}) })
export class WorkspaceCalendarSyncCommandsModule {} export class CalendarCommandsModule {}
@@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { GoogleCalendarSyncCommand } from 'src/modules/calendar/commands/google-calendar-sync.command';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { GoogleCalendarSyncCronCommand } from 'src/modules/calendar/crons/commands/google-calendar-sync.cron.command';
@Module({
providers: [GoogleCalendarSyncCronCommand],
})
export class CalendarCronCommandsModule {}
@@ -4,14 +4,15 @@ import { Command, CommandRunner } from 'nest-commander';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { GoogleCalendarSyncCronJob } from 'src/modules/calendar/jobs/crons/google-calendar-sync.cron.job'; import { GoogleCalendarSyncCronJob } from 'src/modules/calendar/crons/jobs/google-calendar-sync.cron.job';
import { googleCalendarSyncCronPattern } from 'src/modules/calendar/jobs/crons/pattern/google-calendar-sync.cron.pattern';
const GOOGLE_CALENDAR_SYNC_CRON_PATTERN = '*/5 * * * *';
@Command({ @Command({
name: 'cron:calendar:google-calendar-sync', name: 'cron:calendar:google-calendar-sync',
description: 'Starts a cron job to sync google calendar for all workspaces.', description: 'Starts a cron job to sync google calendar for all workspaces.',
}) })
export class StartGoogleCalendarSyncCronJobCommand extends CommandRunner { export class GoogleCalendarSyncCronCommand extends CommandRunner {
constructor( constructor(
@Inject(MessageQueue.cronQueue) @Inject(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService, private readonly messageQueueService: MessageQueueService,
@@ -24,7 +25,7 @@ export class StartGoogleCalendarSyncCronJobCommand extends CommandRunner {
GoogleCalendarSyncCronJob.name, GoogleCalendarSyncCronJob.name,
undefined, undefined,
{ {
repeat: { pattern: googleCalendarSyncCronPattern }, repeat: { pattern: GOOGLE_CALENDAR_SYNC_CRON_PATTERN },
}, },
); );
} }
@@ -1 +0,0 @@
export const googleCalendarSyncCronPattern = '*/5 * * * *';
@@ -4,7 +4,7 @@ import { Command, CommandRunner } from 'nest-commander';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/jobs/crons/fetch-all-messages-from-cache.cron.job'; import { GmailFetchMessagesFromCacheCronJob } from 'src/modules/messaging/crons/jobs/gmail-fetch-messages-from-cache.cron.job';
@Command({ @Command({
name: 'cron:messaging:gmail-fetch-messages-from-cache', name: 'cron:messaging:gmail-fetch-messages-from-cache',
@@ -20,7 +20,7 @@ export class GmailFetchMessagesFromCacheCronCommand extends CommandRunner {
async run(): Promise<void> { async run(): Promise<void> {
await this.messageQueueService.addCron<undefined>( await this.messageQueueService.addCron<undefined>(
FetchAllMessagesFromCacheCronJob.name, GmailFetchMessagesFromCacheCronJob.name,
undefined, undefined,
{ {
repeat: { repeat: {
@@ -4,8 +4,9 @@ import { Command, CommandRunner } from 'nest-commander';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { GmailPartialSyncCronJob } from 'src/modules/messaging/jobs/crons/gmail-partial-sync.cron.job'; import { GmailPartialSyncCronJob } from 'src/modules/messaging/crons/jobs/gmail-partial-sync.cron.job';
import { fetchAllWorkspacesMessagesCronPattern } from 'src/modules/messaging/jobs/crons/patterns/fetch-all-workspaces-messages.cron.pattern';
const GMAIL_PARTIAL_SYNC_CRON_PATTERN = '*/5 * * * *';
@Command({ @Command({
name: 'cron:messaging:gmail-partial-sync', name: 'cron:messaging:gmail-partial-sync',
@@ -25,7 +26,7 @@ export class GmailPartialSyncCronCommand extends CommandRunner {
GmailPartialSyncCronJob.name, GmailPartialSyncCronJob.name,
undefined, undefined,
{ {
repeat: { pattern: fetchAllWorkspacesMessagesCronPattern }, repeat: { pattern: GMAIL_PARTIAL_SYNC_CRON_PATTERN },
}, },
); );
} }
@@ -2,8 +2,8 @@ import { Module } from '@nestjs/common';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module'; import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata'; import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { GmailFetchMessagesFromCacheCronCommand } from 'src/modules/messaging/jobs/crons/gmail-fetch-messages-from-cache.cron.command'; import { GmailFetchMessagesFromCacheCronCommand } from 'src/modules/messaging/crons/commands/gmail-fetch-messages-from-cache.cron.command';
import { GmailPartialSyncCronCommand } from 'src/modules/messaging/jobs/crons/gmail-partial-sync.cron.command'; import { GmailPartialSyncCronCommand } from 'src/modules/messaging/crons/commands/gmail-partial-sync.cron.command';
@Module({ @Module({
imports: [ imports: [
ObjectMetadataRepositoryModule.forFeature([ConnectedAccountObjectMetadata]), ObjectMetadataRepositoryModule.forFeature([ConnectedAccountObjectMetadata]),
@@ -13,4 +13,4 @@ import { GmailPartialSyncCronCommand } from 'src/modules/messaging/jobs/crons/gm
GmailFetchMessagesFromCacheCronCommand, GmailFetchMessagesFromCacheCronCommand,
], ],
}) })
export class MessagingCommandModule {} export class MessagingCronCommandsModule {}
@@ -13,7 +13,7 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj
import { GmailFetchMessageContentFromCacheService } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service'; import { GmailFetchMessageContentFromCacheService } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service';
@Injectable() @Injectable()
export class FetchAllMessagesFromCacheCronJob export class GmailFetchMessagesFromCacheCronJob
implements MessageQueueJob<undefined> implements MessageQueueJob<undefined>
{ {
constructor( constructor(
@@ -13,9 +13,9 @@ import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-s
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata'; import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { import {
GmailPartialSyncV2Job as GmailPartialSyncV2Job, GmailPartialSyncJob as GmailPartialSyncJob,
GmailPartialSyncV2JobData as GmailPartialSyncV2JobData, GmailPartialSyncJobData as GmailPartialSyncJobData,
} from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job'; } from 'src/modules/messaging/jobs/gmail-partial-sync.job';
@Injectable() @Injectable()
export class GmailPartialSyncCronJob implements MessageQueueJob<undefined> { export class GmailPartialSyncCronJob implements MessageQueueJob<undefined> {
@@ -63,8 +63,8 @@ export class GmailPartialSyncCronJob implements MessageQueueJob<undefined> {
await this.connectedAccountRepository.getAll(workspaceId); await this.connectedAccountRepository.getAll(workspaceId);
for (const connectedAccount of connectedAccounts) { for (const connectedAccount of connectedAccounts) {
await this.messageQueueService.add<GmailPartialSyncV2JobData>( await this.messageQueueService.add<GmailPartialSyncJobData>(
GmailPartialSyncV2Job.name, GmailPartialSyncJob.name,
{ {
workspaceId, workspaceId,
connectedAccountId: connectedAccount.id, connectedAccountId: connectedAccount.id,
@@ -1 +0,0 @@
export const fetchAllWorkspacesMessagesCronPattern = '*/5 * * * *';
@@ -5,23 +5,21 @@ import { MessageQueueJob } from 'src/engine/integrations/message-queue/interface
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service'; import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service'; import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
export type GmailFullSyncV2JobData = { export type GmailFullSyncJobData = {
workspaceId: string; workspaceId: string;
connectedAccountId: string; connectedAccountId: string;
}; };
@Injectable() @Injectable()
export class GmailFullSyncV2Job export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
implements MessageQueueJob<GmailFullSyncV2JobData> private readonly logger = new Logger(GmailFullSyncJob.name);
{
private readonly logger = new Logger(GmailFullSyncV2Job.name);
constructor( constructor(
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService, private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
private readonly gmailFullSyncV2Service: GmailFullSyncV2Service, private readonly gmailFullSyncV2Service: GmailFullSyncV2Service,
) {} ) {}
async handle(data: GmailFullSyncV2JobData): Promise<void> { async handle(data: GmailFullSyncJobData): Promise<void> {
this.logger.log( this.logger.log(
`gmail full-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`, `gmail full-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
); );
@@ -5,23 +5,23 @@ import { MessageQueueJob } from 'src/engine/integrations/message-queue/interface
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service'; import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service'; import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
export type GmailPartialSyncV2JobData = { export type GmailPartialSyncJobData = {
workspaceId: string; workspaceId: string;
connectedAccountId: string; connectedAccountId: string;
}; };
@Injectable() @Injectable()
export class GmailPartialSyncV2Job export class GmailPartialSyncJob
implements MessageQueueJob<GmailPartialSyncV2JobData> implements MessageQueueJob<GmailPartialSyncJobData>
{ {
private readonly logger = new Logger(GmailPartialSyncV2Job.name); private readonly logger = new Logger(GmailPartialSyncJob.name);
constructor( constructor(
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService, private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
private readonly gmailPartialSyncV2Service: GmailPartialSyncV2Service, private readonly gmailPartialSyncV2Service: GmailPartialSyncV2Service,
) {} ) {}
async handle(data: GmailPartialSyncV2JobData): Promise<void> { async handle(data: GmailPartialSyncJobData): Promise<void> {
this.logger.log( this.logger.log(
`gmail partial-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`, `gmail partial-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
); );
@@ -19,9 +19,9 @@ import { GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/const
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { SaveMessageAndEmitContactCreationEventService } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.service'; import { SaveMessageAndEmitContactCreationEventService } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.service';
import { import {
GmailFullSyncV2JobData, GmailFullSyncJobData,
GmailFullSyncV2Job, GmailFullSyncJob,
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job'; } from 'src/modules/messaging/jobs/gmail-full-sync.job';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { GMAIL_ONGOING_SYNC_TIMEOUT } from 'src/modules/messaging/constants/gmail-ongoing-sync-timeout.constant'; import { GMAIL_ONGOING_SYNC_TIMEOUT } from 'src/modules/messaging/constants/gmail-ongoing-sync-timeout.constant';
@@ -265,8 +265,8 @@ export class GmailFetchMessageContentFromCacheService {
workspaceId: string, workspaceId: string,
connectedAccountId: string, connectedAccountId: string,
) { ) {
await this.messageQueueService.add<GmailFullSyncV2JobData>( await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncV2Job.name, GmailFullSyncJob.name,
{ workspaceId, connectedAccountId }, { workspaceId, connectedAccountId },
); );
} }
@@ -21,9 +21,9 @@ import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decora
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { import {
GmailFullSyncV2Job, GmailFullSyncJob,
GmailFullSyncV2JobData, GmailFullSyncJobData,
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job'; } from 'src/modules/messaging/jobs/gmail-full-sync.job';
@Injectable() @Injectable()
export class GmailPartialSyncV2Service { export class GmailPartialSyncV2Service {
@@ -330,8 +330,8 @@ export class GmailPartialSyncV2Service {
workspaceId: string, workspaceId: string,
connectedAccountId: string, connectedAccountId: string,
) { ) {
await this.messageQueueService.add<GmailFullSyncV2JobData>( await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncV2Job.name, GmailFullSyncJob.name,
{ workspaceId, connectedAccountId }, { workspaceId, connectedAccountId },
); );
} }