[messaing] improve messaging import (#4650)
* [messaging] improve full-sync fetching strategy * fix * rebase * fix * fix * fix rebase * fix * fix * fix * fix * fix * remove deletion * fix setPop with memory storage * fix pgBoss and remove unnecessary job * fix throw * fix * add timeout to ongoing sync
This commit is contained in:
Weiko
@ -1,78 +0,0 @@
|
||||
import { Cache } from '@nestjs/cache-manager';
|
||||
|
||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
|
||||
const cacheStorageNamespace = CacheStorageNamespace.Messaging;
|
||||
|
||||
describe('CacheStorageService', () => {
|
||||
let cacheStorageService: CacheStorageService;
|
||||
let cacheManagerMock: Partial<Cache>;
|
||||
|
||||
beforeEach(() => {
|
||||
cacheManagerMock = {
|
||||
get: jest.fn(),
|
||||
set: jest.fn(),
|
||||
};
|
||||
|
||||
cacheStorageService = new CacheStorageService(
|
||||
cacheManagerMock as Cache,
|
||||
cacheStorageNamespace,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('get', () => {
|
||||
it('should call cacheManager.get with the correct namespaced key', async () => {
|
||||
const key = 'testKey';
|
||||
const namespacedKey = `${cacheStorageNamespace}:${key}`;
|
||||
|
||||
await cacheStorageService.get(key);
|
||||
|
||||
expect(cacheManagerMock.get).toHaveBeenCalledWith(namespacedKey);
|
||||
});
|
||||
|
||||
it('should return the value returned by cacheManager.get', async () => {
|
||||
const key = 'testKey';
|
||||
const value = 'testValue';
|
||||
|
||||
jest.spyOn(cacheManagerMock, 'get').mockResolvedValue(value);
|
||||
|
||||
const result = await cacheStorageService.get(key);
|
||||
|
||||
expect(result).toBe(value);
|
||||
});
|
||||
});
|
||||
|
||||
describe('set', () => {
|
||||
it('should call cacheManager.set with the correct namespaced key, value, and optional ttl', async () => {
|
||||
const key = 'testKey';
|
||||
const value = 'testValue';
|
||||
const ttl = 60;
|
||||
const namespacedKey = `${cacheStorageNamespace}:${key}`;
|
||||
|
||||
await cacheStorageService.set(key, value, ttl);
|
||||
|
||||
expect(cacheManagerMock.set).toHaveBeenCalledWith(
|
||||
namespacedKey,
|
||||
value,
|
||||
ttl,
|
||||
);
|
||||
});
|
||||
|
||||
it('should not throw if cacheManager.set resolves successfully', async () => {
|
||||
const key = 'testKey';
|
||||
const value = 'testValue';
|
||||
const ttl = 60;
|
||||
|
||||
jest.spyOn(cacheManagerMock, 'set').mockResolvedValue(undefined);
|
||||
|
||||
await expect(
|
||||
cacheStorageService.set(key, value, ttl),
|
||||
).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -1,25 +1,67 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
|
||||
|
||||
import { RedisCache } from 'cache-manager-redis-yet';
|
||||
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
|
||||
@Injectable()
|
||||
export class CacheStorageService {
|
||||
constructor(
|
||||
@Inject(CACHE_MANAGER)
|
||||
private readonly cacheManager: Cache,
|
||||
private readonly cache: Cache,
|
||||
private readonly namespace: CacheStorageNamespace,
|
||||
) {}
|
||||
|
||||
async get<T>(key: string): Promise<T | undefined> {
|
||||
return this.cacheManager.get(`${this.namespace}:${key}`);
|
||||
return this.cache.get(`${this.namespace}:${key}`);
|
||||
}
|
||||
|
||||
async set<T>(key: string, value: T, ttl?: number) {
|
||||
return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
|
||||
return this.cache.set(`${this.namespace}:${key}`, value, ttl);
|
||||
}
|
||||
|
||||
async del(key: string) {
|
||||
return this.cacheManager.del(`${this.namespace}:${key}`);
|
||||
return this.cache.del(`${this.namespace}:${key}`);
|
||||
}
|
||||
|
||||
async setAdd(key: string, value: string[]) {
|
||||
if (value.length === 0) {
|
||||
return;
|
||||
}
|
||||
if (this.isRedisCache()) {
|
||||
return (this.cache as RedisCache).store.client.sAdd(
|
||||
`${this.namespace}:${key}`,
|
||||
value,
|
||||
);
|
||||
}
|
||||
this.get(key).then((res: string[]) => {
|
||||
if (res) {
|
||||
this.set(key, [...res, ...value]);
|
||||
} else {
|
||||
this.set(key, value);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async setPop(key: string, size: number = 1) {
|
||||
if (this.isRedisCache()) {
|
||||
return (this.cache as RedisCache).store.client.sPop(
|
||||
`${this.namespace}:${key}`,
|
||||
size,
|
||||
);
|
||||
}
|
||||
|
||||
return this.get(key).then((res: string[]) => {
|
||||
if (res) {
|
||||
this.set(key, res.slice(0, -size));
|
||||
}
|
||||
|
||||
return res;
|
||||
});
|
||||
}
|
||||
|
||||
private isRedisCache() {
|
||||
return (this.cache.store as any)?.name === 'redis';
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
import { Inject } from '@nestjs/common';
|
||||
|
||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||
|
||||
export const InjectCacheStorage = (
|
||||
cacheStorageNamespace: CacheStorageNamespace,
|
||||
) => {
|
||||
return Inject(cacheStorageNamespace);
|
||||
};
|
||||
@ -0,0 +1,7 @@
|
||||
import { Inject } from '@nestjs/common';
|
||||
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
|
||||
export const InjectMessageQueue = (messageQueueName: MessageQueue) => {
|
||||
return Inject(messageQueueName);
|
||||
};
|
||||
@ -1,6 +1,9 @@
|
||||
import { Queue, QueueOptions, Worker } from 'bullmq';
|
||||
|
||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
import {
|
||||
QueueCronJobOptions,
|
||||
QueueJobOptions,
|
||||
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
|
||||
@ -53,8 +56,7 @@ export class BullMQDriver implements MessageQueueDriver {
|
||||
queueName: MessageQueue,
|
||||
jobName: string,
|
||||
data: T,
|
||||
pattern: string,
|
||||
options?: QueueJobOptions,
|
||||
options?: QueueCronJobOptions,
|
||||
): Promise<void> {
|
||||
if (!this.queueMap[queueName]) {
|
||||
throw new Error(
|
||||
@ -64,9 +66,7 @@ export class BullMQDriver implements MessageQueueDriver {
|
||||
const queueOptions = {
|
||||
jobId: options?.id,
|
||||
priority: options?.priority,
|
||||
repeat: {
|
||||
pattern,
|
||||
},
|
||||
repeat: options?.repeat,
|
||||
};
|
||||
|
||||
await this.queueMap[queueName].add(jobName, data, queueOptions);
|
||||
|
||||
@ -3,3 +3,11 @@ export interface QueueJobOptions {
|
||||
priority?: number;
|
||||
retryLimit?: number;
|
||||
}
|
||||
|
||||
export interface QueueCronJobOptions extends QueueJobOptions {
|
||||
repeat?: {
|
||||
every?: number;
|
||||
pattern?: string;
|
||||
limit?: number;
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
import {
|
||||
QueueCronJobOptions,
|
||||
QueueJobOptions,
|
||||
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
@ -18,8 +21,7 @@ export interface MessageQueueDriver {
|
||||
queueName: MessageQueue,
|
||||
jobName: string,
|
||||
data: T,
|
||||
pattern: string,
|
||||
options?: QueueJobOptions,
|
||||
options?: QueueCronJobOptions,
|
||||
);
|
||||
removeCron(queueName: MessageQueue, jobName: string, pattern?: string);
|
||||
stop?(): Promise<void>;
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import PgBoss from 'pg-boss';
|
||||
|
||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
import {
|
||||
QueueCronJobOptions,
|
||||
QueueJobOptions,
|
||||
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
|
||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||
|
||||
@ -8,6 +11,8 @@ import { MessageQueueDriver } from './interfaces/message-queue-driver.interface'
|
||||
|
||||
export type PgBossDriverOptions = PgBoss.ConstructorOptions;
|
||||
|
||||
const DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED = '*/10 * * * *';
|
||||
|
||||
export class PgBossDriver implements MessageQueueDriver {
|
||||
private pgBoss: PgBoss;
|
||||
|
||||
@ -34,16 +39,15 @@ export class PgBossDriver implements MessageQueueDriver {
|
||||
queueName: MessageQueue,
|
||||
jobName: string,
|
||||
data: T,
|
||||
pattern: string,
|
||||
options?: QueueJobOptions,
|
||||
options?: QueueCronJobOptions,
|
||||
): Promise<void> {
|
||||
await this.pgBoss.schedule(
|
||||
`${queueName}.${jobName}`,
|
||||
pattern,
|
||||
options?.repeat?.pattern ??
|
||||
DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED,
|
||||
data as object,
|
||||
options
|
||||
? {
|
||||
...options,
|
||||
singletonKey: options?.id,
|
||||
}
|
||||
: {},
|
||||
|
||||
@ -33,9 +33,8 @@ export class SyncDriver implements MessageQueueDriver {
|
||||
_queueName: MessageQueue,
|
||||
jobName: string,
|
||||
data: T,
|
||||
pattern: string,
|
||||
): Promise<void> {
|
||||
this.logger.log(`Running '${pattern}' cron job with SyncDriver`);
|
||||
this.logger.log(`Running cron job with SyncDriver`);
|
||||
|
||||
const jobClassName = getJobClassName(jobName);
|
||||
const job: MessageQueueCronJobData<MessageQueueJobData | undefined> =
|
||||
|
||||
@ -44,9 +44,15 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
|
||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||
import { MessageParticipantObjectMetadata } from 'src/modules/messaging/standard-objects/message-participant.object-metadata';
|
||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
|
||||
import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
|
||||
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
|
||||
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
|
||||
import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module';
|
||||
import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module';
|
||||
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/commands/crons/fetch-all-messages-from-cache.cron-job';
|
||||
import { GmailFullSyncV2Job } from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||
import { GmailPartialSyncV2Job } from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job';
|
||||
import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@ -78,6 +84,9 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
|
||||
MessageChannelObjectMetadata,
|
||||
EventObjectMetadata,
|
||||
]),
|
||||
GmailFullSynV2Module,
|
||||
GmailFetchMessageContentFromCacheModule,
|
||||
GmailPartialSyncV2Module,
|
||||
],
|
||||
providers: [
|
||||
{
|
||||
@ -142,6 +151,18 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
|
||||
provide: SaveEventToDbJob.name,
|
||||
useClass: SaveEventToDbJob,
|
||||
},
|
||||
{
|
||||
provide: FetchAllMessagesFromCacheCronJob.name,
|
||||
useClass: FetchAllMessagesFromCacheCronJob,
|
||||
},
|
||||
{
|
||||
provide: GmailFullSyncV2Job.name,
|
||||
useClass: GmailFullSyncV2Job,
|
||||
},
|
||||
{
|
||||
provide: GmailPartialSyncV2Job.name,
|
||||
useClass: GmailPartialSyncV2Job,
|
||||
},
|
||||
],
|
||||
})
|
||||
export class JobsModule {
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
|
||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
import {
|
||||
QueueCronJobOptions,
|
||||
QueueJobOptions,
|
||||
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
|
||||
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||
|
||||
@ -37,10 +40,9 @@ export class MessageQueueService implements OnModuleDestroy {
|
||||
addCron<T extends MessageQueueJobData | undefined>(
|
||||
jobName: string,
|
||||
data: T,
|
||||
pattern: string,
|
||||
options?: QueueJobOptions,
|
||||
options?: QueueCronJobOptions,
|
||||
): Promise<void> {
|
||||
return this.driver.addCron(this.queueName, jobName, data, pattern, options);
|
||||
return this.driver.addCron(this.queueName, jobName, data, options);
|
||||
}
|
||||
|
||||
removeCron(jobName: string, pattern: string): Promise<void> {
|
||||
|
||||
Reference in New Issue
Block a user