admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[messaing] improve messaging import (#4650)

Browse Source 
* [messaging] improve full-sync fetching strategy

* fix

* rebase

* fix

* fix

* fix rebase

* fix

* fix

* fix

* fix

* fix

* remove deletion

* fix setPop with memory storage

* fix pgBoss and remove unnecessary job

* fix throw

* fix

* add timeout to ongoing sync
This commit is contained in:
Weiko
2024-03-27 12:44:03 +01:00
committed by GitHub
parent 5c0b65eecb
commit 5c40e3608b
48 changed files with 1728 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/twenty-server/src/engine/api/graphql/workspace-schema-storage/workspace-schema-storage.service.ts
View File

@ -1,6 +1,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
@ -8,7 +9,7 @@ import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/worksp
@Injectable()
export class WorkspaceSchemaStorageService {
constructor(
@Inject(CacheStorageNamespace.WorkspaceSchema)
@InjectCacheStorage(CacheStorageNamespace.WorkspaceSchema)
private readonly workspaceSchemaCache: CacheStorageService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,

40
packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts
View File

@ -22,6 +22,10 @@ import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import {
GmailFullSyncV2Job,
GmailFullSyncV2JobData,
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
@Injectable()
export class GoogleAPIsService {
@ -75,6 +79,12 @@ export class GoogleAPIsService {
value: true,
});
const isFullSyncV2Enabled = await this.featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKeys.IsFullSyncV2Enabled,
value: true,
});
await workspaceDataSource?.transaction(async (manager) => {
await manager.query(
`INSERT INTO ${dataSourceMetadata.schema}."connectedAccount" ("id", "handle", "provider", "accessToken", "refreshToken", "accountOwnerId") VALUES ($1, $2, $3, $4, $5, $6)`,
@ -107,16 +117,26 @@ export class GoogleAPIsService {
});
if (this.environmentService.get('MESSAGING_PROVIDER_GMAIL_ENABLED')) {
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
{
workspaceId,
connectedAccountId,
},
{
retryLimit: 2,
},
);
if (isFullSyncV2Enabled) {
await this.messageQueueService.add<GmailFullSyncV2JobData>(
GmailFullSyncV2Job.name,
{
workspaceId,
connectedAccountId,
},
);
} else {
await this.messageQueueService.add<GmailFullSyncJobData>(
GmailFullSyncJob.name,
{
workspaceId,
connectedAccountId,
},
{
retryLimit: 2,
},
);
}
}
if (

1
packages/twenty-server/src/engine/core-modules/feature-flag/feature-flag.entity.ts
View File

@ -19,6 +19,7 @@ export enum FeatureFlagKeys {
IsEventObjectEnabled = 'IS_EVENT_OBJECT_ENABLED',
IsAirtableIntegrationEnabled = 'IS_AIRTABLE_INTEGRATION_ENABLED',
IsPostgreSQLIntegrationEnabled = 'IS_POSTGRESQL_INTEGRATION_ENABLED',
IsFullSyncV2Enabled = 'IS_FULL_SYNC_V2_ENABLED',
}
@Entity({ name: 'featureFlag', schema: 'core' })

78
packages/twenty-server/src/engine/integrations/cache-storage/__tests__/cache-storage.service.spec.ts
View File

@ -1,78 +0,0 @@
import { Cache } from '@nestjs/cache-manager';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
const cacheStorageNamespace = CacheStorageNamespace.Messaging;
describe('CacheStorageService', () => {
let cacheStorageService: CacheStorageService;
let cacheManagerMock: Partial<Cache>;
beforeEach(() => {
cacheManagerMock = {
get: jest.fn(),
set: jest.fn(),
};
cacheStorageService = new CacheStorageService(
cacheManagerMock as Cache,
cacheStorageNamespace,
);
});
afterEach(() => {
jest.clearAllMocks();
});
describe('get', () => {
it('should call cacheManager.get with the correct namespaced key', async () => {
const key = 'testKey';
const namespacedKey = `${cacheStorageNamespace}:${key}`;
await cacheStorageService.get(key);
expect(cacheManagerMock.get).toHaveBeenCalledWith(namespacedKey);
});
it('should return the value returned by cacheManager.get', async () => {
const key = 'testKey';
const value = 'testValue';
jest.spyOn(cacheManagerMock, 'get').mockResolvedValue(value);
const result = await cacheStorageService.get(key);
expect(result).toBe(value);
});
});
describe('set', () => {
it('should call cacheManager.set with the correct namespaced key, value, and optional ttl', async () => {
const key = 'testKey';
const value = 'testValue';
const ttl = 60;
const namespacedKey = `${cacheStorageNamespace}:${key}`;
await cacheStorageService.set(key, value, ttl);
expect(cacheManagerMock.set).toHaveBeenCalledWith(
namespacedKey,
value,
ttl,
);
});
it('should not throw if cacheManager.set resolves successfully', async () => {
const key = 'testKey';
const value = 'testValue';
const ttl = 60;
jest.spyOn(cacheManagerMock, 'set').mockResolvedValue(undefined);
await expect(
cacheStorageService.set(key, value, ttl),
).resolves.not.toThrow();
});
});
});

50
packages/twenty-server/src/engine/integrations/cache-storage/cache-storage.service.ts
View File

@ -1,25 +1,67 @@
import { Inject, Injectable } from '@nestjs/common';
import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
import { RedisCache } from 'cache-manager-redis-yet';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
@Injectable()
export class CacheStorageService {
constructor(
@Inject(CACHE_MANAGER)
private readonly cacheManager: Cache,
private readonly cache: Cache,
private readonly namespace: CacheStorageNamespace,
) {}
async get<T>(key: string): Promise<T | undefined> {
return this.cacheManager.get(`${this.namespace}:${key}`);
return this.cache.get(`${this.namespace}:${key}`);
}
async set<T>(key: string, value: T, ttl?: number) {
return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
return this.cache.set(`${this.namespace}:${key}`, value, ttl);
}
async del(key: string) {
return this.cacheManager.del(`${this.namespace}:${key}`);
return this.cache.del(`${this.namespace}:${key}`);
}
async setAdd(key: string, value: string[]) {
if (value.length === 0) {
return;
}
if (this.isRedisCache()) {
return (this.cache as RedisCache).store.client.sAdd(
`${this.namespace}:${key}`,
value,
);
}
this.get(key).then((res: string[]) => {
if (res) {
this.set(key, [...res, ...value]);
} else {
this.set(key, value);
}
});
}
async setPop(key: string, size: number = 1) {
if (this.isRedisCache()) {
return (this.cache as RedisCache).store.client.sPop(
`${this.namespace}:${key}`,
size,
);
}
return this.get(key).then((res: string[]) => {
if (res) {
this.set(key, res.slice(0, -size));
}
return res;
});
}
private isRedisCache() {
return (this.cache.store as any)?.name === 'redis';
}
}

9
packages/twenty-server/src/engine/integrations/cache-storage/decorators/cache-storage.decorator.ts Normal file
View File

@ -0,0 +1,9 @@
import { Inject } from '@nestjs/common';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
export const InjectCacheStorage = (
cacheStorageNamespace: CacheStorageNamespace,
) => {
return Inject(cacheStorageNamespace);
};

7
packages/twenty-server/src/engine/integrations/message-queue/decorators/message-queue.decorator.ts Normal file
View File

@ -0,0 +1,7 @@
import { Inject } from '@nestjs/common';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
export const InjectMessageQueue = (messageQueueName: MessageQueue) => {
return Inject(messageQueueName);
};

12
packages/twenty-server/src/engine/integrations/message-queue/drivers/bullmq.driver.ts
View File

@ -1,6 +1,9 @@
import { Queue, QueueOptions, Worker } from 'bullmq';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
@ -53,8 +56,7 @@ export class BullMQDriver implements MessageQueueDriver {
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
): Promise<void> {
if (!this.queueMap[queueName]) {
throw new Error(
@ -64,9 +66,7 @@ export class BullMQDriver implements MessageQueueDriver {
const queueOptions = {
jobId: options?.id,
priority: options?.priority,
repeat: {
pattern,
},
repeat: options?.repeat,
};
await this.queueMap[queueName].add(jobName, data, queueOptions);

8
packages/twenty-server/src/engine/integrations/message-queue/drivers/interfaces/job-options.interface.ts
View File

@ -3,3 +3,11 @@ export interface QueueJobOptions {
priority?: number;
retryLimit?: number;
}
export interface QueueCronJobOptions extends QueueJobOptions {
repeat?: {
every?: number;
pattern?: string;
limit?: number;
};
}

8
packages/twenty-server/src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface.ts
View File

@ -1,4 +1,7 @@
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
@ -18,8 +21,7 @@ export interface MessageQueueDriver {
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
);
removeCron(queueName: MessageQueue, jobName: string, pattern?: string);
stop?(): Promise<void>;

14
packages/twenty-server/src/engine/integrations/message-queue/drivers/pg-boss.driver.ts
View File

@ -1,6 +1,9 @@
import PgBoss from 'pg-boss';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
@ -8,6 +11,8 @@ import { MessageQueueDriver } from './interfaces/message-queue-driver.interface'
export type PgBossDriverOptions = PgBoss.ConstructorOptions;
const DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED = '*/10 * * * *';
export class PgBossDriver implements MessageQueueDriver {
private pgBoss: PgBoss;
@ -34,16 +39,15 @@ export class PgBossDriver implements MessageQueueDriver {
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
): Promise<void> {
await this.pgBoss.schedule(
`${queueName}.${jobName}`,
pattern,
options?.repeat?.pattern ??
DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED,
data as object,
options
? {
...options,
singletonKey: options?.id,
}
: {},

3
packages/twenty-server/src/engine/integrations/message-queue/drivers/sync.driver.ts
View File

@ -33,9 +33,8 @@ export class SyncDriver implements MessageQueueDriver {
_queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
): Promise<void> {
this.logger.log(`Running '${pattern}' cron job with SyncDriver`);
this.logger.log(`Running cron job with SyncDriver`);
const jobClassName = getJobClassName(jobName);
const job: MessageQueueCronJobData<MessageQueueJobData | undefined> =

23
packages/twenty-server/src/engine/integrations/message-queue/jobs.module.ts
View File

@ -44,9 +44,15 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { MessageParticipantObjectMetadata } from 'src/modules/messaging/standard-objects/message-participant.object-metadata';
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module';
import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module';
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/commands/crons/fetch-all-messages-from-cache.cron-job';
import { GmailFullSyncV2Job } from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
import { GmailPartialSyncV2Job } from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job';
import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module';
@Module({
imports: [
@ -78,6 +84,9 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
MessageChannelObjectMetadata,
EventObjectMetadata,
]),
GmailFullSynV2Module,
GmailFetchMessageContentFromCacheModule,
GmailPartialSyncV2Module,
],
providers: [
{
@ -142,6 +151,18 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
provide: SaveEventToDbJob.name,
useClass: SaveEventToDbJob,
},
{
provide: FetchAllMessagesFromCacheCronJob.name,
useClass: FetchAllMessagesFromCacheCronJob,
},
{
provide: GmailFullSyncV2Job.name,
useClass: GmailFullSyncV2Job,
},
{
provide: GmailPartialSyncV2Job.name,
useClass: GmailPartialSyncV2Job,
},
],
})
export class JobsModule {

10
packages/twenty-server/src/engine/integrations/message-queue/services/message-queue.service.ts
View File

@ -1,6 +1,9 @@
import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import {
QueueCronJobOptions,
QueueJobOptions,
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
@ -37,10 +40,9 @@ export class MessageQueueService implements OnModuleDestroy {
addCron<T extends MessageQueueJobData | undefined>(
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
options?: QueueCronJobOptions,
): Promise<void> {
return this.driver.addCron(this.queueName, jobName, data, pattern, options);
return this.driver.addCron(this.queueName, jobName, data, options);
}
removeCron(jobName: string, pattern: string): Promise<void> {

3
packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/start-clean-inactive-workspaces.cron.command.ts
View File

@ -23,8 +23,7 @@ export class StartCleanInactiveWorkspacesCronCommand extends CommandRunner {
await this.messageQueueService.addCron<undefined>(
CleanInactiveWorkspaceJob.name,
undefined,
cleanInactiveWorkspaceCronPattern,
{ retryLimit: 3 },
{ retryLimit: 3, repeat: { pattern: cleanInactiveWorkspaceCronPattern } },
);
}
}

2
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/commands/add-standard-id.command.ts
View File

@ -58,6 +58,7 @@ export class AddStandardIdCommand extends CommandRunner {
IS_EVENT_OBJECT_ENABLED: true,
IS_AIRTABLE_INTEGRATION_ENABLED: true,
IS_POSTGRESQL_INTEGRATION_ENABLED: true,
IS_FULL_SYNC_V2_ENABLED: false,
},
);
const standardFieldMetadataCollection = this.standardFieldFactory.create(
@ -72,6 +73,7 @@ export class AddStandardIdCommand extends CommandRunner {
IS_EVENT_OBJECT_ENABLED: true,
IS_AIRTABLE_INTEGRATION_ENABLED: true,
IS_POSTGRESQL_INTEGRATION_ENABLED: true,
IS_FULL_SYNC_V2_ENABLED: false,
},
);

4
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids.ts
View File

@ -169,6 +169,10 @@ export const messageChannelStandardFieldIds = {
type: '20202020-ae95-42d9-a3f1-797a2ea22122',
isContactAutoCreationEnabled: '20202020-fabd-4f14-b7c6-3310f6d132c6',
messageChannelMessageAssociations: '20202020-49b8-4766-88fd-75f1e21b3d5f',
syncExternalId: '20202020-79d1-41cf-b738-bcf5ed61e256',
syncedAt: '20202020-263d-4c6b-ad51-137ada56f7d4',
syncStatus: '20202020-56a1-4f7e-9880-a8493bb899cc',
ongoingSyncStartedAt: '20202020-8c61-4a42-ae63-73c1c3c52e06',
};
export const messageParticipantStandardFieldIds = {