[messaing] improve messaging import (#4650)
* [messaging] improve full-sync fetching strategy * fix * rebase * fix * fix * fix rebase * fix * fix * fix * fix * fix * remove deletion * fix setPop with memory storage * fix pgBoss and remove unnecessary job * fix throw * fix * add timeout to ongoing sync
This commit is contained in:
@ -23,7 +23,11 @@ export class StartDataSeedDemoWorkspaceCronCommand extends CommandRunner {
|
|||||||
await this.messageQueueService.addCron<undefined>(
|
await this.messageQueueService.addCron<undefined>(
|
||||||
DataSeedDemoWorkspaceJob.name,
|
DataSeedDemoWorkspaceJob.name,
|
||||||
undefined,
|
undefined,
|
||||||
dataSeedDemoWorkspaceCronPattern,
|
{
|
||||||
|
repeat: {
|
||||||
|
pattern: dataSeedDemoWorkspaceCronPattern,
|
||||||
|
},
|
||||||
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -40,6 +40,11 @@ export const seedFeatureFlags = async (
|
|||||||
workspaceId: workspaceId,
|
workspaceId: workspaceId,
|
||||||
value: true,
|
value: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
workspaceId: workspaceId,
|
||||||
|
value: true,
|
||||||
|
},
|
||||||
])
|
])
|
||||||
.execute();
|
.execute();
|
||||||
};
|
};
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
|
|
||||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||||
|
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||||
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
|
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
|
||||||
@ -8,7 +9,7 @@ import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/worksp
|
|||||||
@Injectable()
|
@Injectable()
|
||||||
export class WorkspaceSchemaStorageService {
|
export class WorkspaceSchemaStorageService {
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(CacheStorageNamespace.WorkspaceSchema)
|
@InjectCacheStorage(CacheStorageNamespace.WorkspaceSchema)
|
||||||
private readonly workspaceSchemaCache: CacheStorageService,
|
private readonly workspaceSchemaCache: CacheStorageService,
|
||||||
|
|
||||||
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
|
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
|
||||||
|
|||||||
@ -22,6 +22,10 @@ import {
|
|||||||
FeatureFlagEntity,
|
FeatureFlagEntity,
|
||||||
FeatureFlagKeys,
|
FeatureFlagKeys,
|
||||||
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||||
|
import {
|
||||||
|
GmailFullSyncV2Job,
|
||||||
|
GmailFullSyncV2JobData,
|
||||||
|
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class GoogleAPIsService {
|
export class GoogleAPIsService {
|
||||||
@ -75,6 +79,12 @@ export class GoogleAPIsService {
|
|||||||
value: true,
|
value: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const isFullSyncV2Enabled = await this.featureFlagRepository.findOneBy({
|
||||||
|
workspaceId,
|
||||||
|
key: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
value: true,
|
||||||
|
});
|
||||||
|
|
||||||
await workspaceDataSource?.transaction(async (manager) => {
|
await workspaceDataSource?.transaction(async (manager) => {
|
||||||
await manager.query(
|
await manager.query(
|
||||||
`INSERT INTO ${dataSourceMetadata.schema}."connectedAccount" ("id", "handle", "provider", "accessToken", "refreshToken", "accountOwnerId") VALUES ($1, $2, $3, $4, $5, $6)`,
|
`INSERT INTO ${dataSourceMetadata.schema}."connectedAccount" ("id", "handle", "provider", "accessToken", "refreshToken", "accountOwnerId") VALUES ($1, $2, $3, $4, $5, $6)`,
|
||||||
@ -107,16 +117,26 @@ export class GoogleAPIsService {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (this.environmentService.get('MESSAGING_PROVIDER_GMAIL_ENABLED')) {
|
if (this.environmentService.get('MESSAGING_PROVIDER_GMAIL_ENABLED')) {
|
||||||
await this.messageQueueService.add<GmailFullSyncJobData>(
|
if (isFullSyncV2Enabled) {
|
||||||
GmailFullSyncJob.name,
|
await this.messageQueueService.add<GmailFullSyncV2JobData>(
|
||||||
{
|
GmailFullSyncV2Job.name,
|
||||||
workspaceId,
|
{
|
||||||
connectedAccountId,
|
workspaceId,
|
||||||
},
|
connectedAccountId,
|
||||||
{
|
},
|
||||||
retryLimit: 2,
|
);
|
||||||
},
|
} else {
|
||||||
);
|
await this.messageQueueService.add<GmailFullSyncJobData>(
|
||||||
|
GmailFullSyncJob.name,
|
||||||
|
{
|
||||||
|
workspaceId,
|
||||||
|
connectedAccountId,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
retryLimit: 2,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
|||||||
@ -19,6 +19,7 @@ export enum FeatureFlagKeys {
|
|||||||
IsEventObjectEnabled = 'IS_EVENT_OBJECT_ENABLED',
|
IsEventObjectEnabled = 'IS_EVENT_OBJECT_ENABLED',
|
||||||
IsAirtableIntegrationEnabled = 'IS_AIRTABLE_INTEGRATION_ENABLED',
|
IsAirtableIntegrationEnabled = 'IS_AIRTABLE_INTEGRATION_ENABLED',
|
||||||
IsPostgreSQLIntegrationEnabled = 'IS_POSTGRESQL_INTEGRATION_ENABLED',
|
IsPostgreSQLIntegrationEnabled = 'IS_POSTGRESQL_INTEGRATION_ENABLED',
|
||||||
|
IsFullSyncV2Enabled = 'IS_FULL_SYNC_V2_ENABLED',
|
||||||
}
|
}
|
||||||
|
|
||||||
@Entity({ name: 'featureFlag', schema: 'core' })
|
@Entity({ name: 'featureFlag', schema: 'core' })
|
||||||
|
|||||||
@ -1,78 +0,0 @@
|
|||||||
import { Cache } from '@nestjs/cache-manager';
|
|
||||||
|
|
||||||
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
|
||||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
|
||||||
|
|
||||||
const cacheStorageNamespace = CacheStorageNamespace.Messaging;
|
|
||||||
|
|
||||||
describe('CacheStorageService', () => {
|
|
||||||
let cacheStorageService: CacheStorageService;
|
|
||||||
let cacheManagerMock: Partial<Cache>;
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
cacheManagerMock = {
|
|
||||||
get: jest.fn(),
|
|
||||||
set: jest.fn(),
|
|
||||||
};
|
|
||||||
|
|
||||||
cacheStorageService = new CacheStorageService(
|
|
||||||
cacheManagerMock as Cache,
|
|
||||||
cacheStorageNamespace,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
jest.clearAllMocks();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('get', () => {
|
|
||||||
it('should call cacheManager.get with the correct namespaced key', async () => {
|
|
||||||
const key = 'testKey';
|
|
||||||
const namespacedKey = `${cacheStorageNamespace}:${key}`;
|
|
||||||
|
|
||||||
await cacheStorageService.get(key);
|
|
||||||
|
|
||||||
expect(cacheManagerMock.get).toHaveBeenCalledWith(namespacedKey);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should return the value returned by cacheManager.get', async () => {
|
|
||||||
const key = 'testKey';
|
|
||||||
const value = 'testValue';
|
|
||||||
|
|
||||||
jest.spyOn(cacheManagerMock, 'get').mockResolvedValue(value);
|
|
||||||
|
|
||||||
const result = await cacheStorageService.get(key);
|
|
||||||
|
|
||||||
expect(result).toBe(value);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('set', () => {
|
|
||||||
it('should call cacheManager.set with the correct namespaced key, value, and optional ttl', async () => {
|
|
||||||
const key = 'testKey';
|
|
||||||
const value = 'testValue';
|
|
||||||
const ttl = 60;
|
|
||||||
const namespacedKey = `${cacheStorageNamespace}:${key}`;
|
|
||||||
|
|
||||||
await cacheStorageService.set(key, value, ttl);
|
|
||||||
|
|
||||||
expect(cacheManagerMock.set).toHaveBeenCalledWith(
|
|
||||||
namespacedKey,
|
|
||||||
value,
|
|
||||||
ttl,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should not throw if cacheManager.set resolves successfully', async () => {
|
|
||||||
const key = 'testKey';
|
|
||||||
const value = 'testValue';
|
|
||||||
const ttl = 60;
|
|
||||||
|
|
||||||
jest.spyOn(cacheManagerMock, 'set').mockResolvedValue(undefined);
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
cacheStorageService.set(key, value, ttl),
|
|
||||||
).resolves.not.toThrow();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@ -1,25 +1,67 @@
|
|||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
|
import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
|
||||||
|
|
||||||
|
import { RedisCache } from 'cache-manager-redis-yet';
|
||||||
|
|
||||||
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class CacheStorageService {
|
export class CacheStorageService {
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(CACHE_MANAGER)
|
@Inject(CACHE_MANAGER)
|
||||||
private readonly cacheManager: Cache,
|
private readonly cache: Cache,
|
||||||
private readonly namespace: CacheStorageNamespace,
|
private readonly namespace: CacheStorageNamespace,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async get<T>(key: string): Promise<T | undefined> {
|
async get<T>(key: string): Promise<T | undefined> {
|
||||||
return this.cacheManager.get(`${this.namespace}:${key}`);
|
return this.cache.get(`${this.namespace}:${key}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
async set<T>(key: string, value: T, ttl?: number) {
|
async set<T>(key: string, value: T, ttl?: number) {
|
||||||
return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
|
return this.cache.set(`${this.namespace}:${key}`, value, ttl);
|
||||||
}
|
}
|
||||||
|
|
||||||
async del(key: string) {
|
async del(key: string) {
|
||||||
return this.cacheManager.del(`${this.namespace}:${key}`);
|
return this.cache.del(`${this.namespace}:${key}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async setAdd(key: string, value: string[]) {
|
||||||
|
if (value.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.isRedisCache()) {
|
||||||
|
return (this.cache as RedisCache).store.client.sAdd(
|
||||||
|
`${this.namespace}:${key}`,
|
||||||
|
value,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
this.get(key).then((res: string[]) => {
|
||||||
|
if (res) {
|
||||||
|
this.set(key, [...res, ...value]);
|
||||||
|
} else {
|
||||||
|
this.set(key, value);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async setPop(key: string, size: number = 1) {
|
||||||
|
if (this.isRedisCache()) {
|
||||||
|
return (this.cache as RedisCache).store.client.sPop(
|
||||||
|
`${this.namespace}:${key}`,
|
||||||
|
size,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.get(key).then((res: string[]) => {
|
||||||
|
if (res) {
|
||||||
|
this.set(key, res.slice(0, -size));
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private isRedisCache() {
|
||||||
|
return (this.cache.store as any)?.name === 'redis';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,9 @@
|
|||||||
|
import { Inject } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||||
|
|
||||||
|
export const InjectCacheStorage = (
|
||||||
|
cacheStorageNamespace: CacheStorageNamespace,
|
||||||
|
) => {
|
||||||
|
return Inject(cacheStorageNamespace);
|
||||||
|
};
|
||||||
@ -0,0 +1,7 @@
|
|||||||
|
import { Inject } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||||
|
|
||||||
|
export const InjectMessageQueue = (messageQueueName: MessageQueue) => {
|
||||||
|
return Inject(messageQueueName);
|
||||||
|
};
|
||||||
@ -1,6 +1,9 @@
|
|||||||
import { Queue, QueueOptions, Worker } from 'bullmq';
|
import { Queue, QueueOptions, Worker } from 'bullmq';
|
||||||
|
|
||||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
import {
|
||||||
|
QueueCronJobOptions,
|
||||||
|
QueueJobOptions,
|
||||||
|
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||||
|
|
||||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||||
|
|
||||||
@ -53,8 +56,7 @@ export class BullMQDriver implements MessageQueueDriver {
|
|||||||
queueName: MessageQueue,
|
queueName: MessageQueue,
|
||||||
jobName: string,
|
jobName: string,
|
||||||
data: T,
|
data: T,
|
||||||
pattern: string,
|
options?: QueueCronJobOptions,
|
||||||
options?: QueueJobOptions,
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (!this.queueMap[queueName]) {
|
if (!this.queueMap[queueName]) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
@ -64,9 +66,7 @@ export class BullMQDriver implements MessageQueueDriver {
|
|||||||
const queueOptions = {
|
const queueOptions = {
|
||||||
jobId: options?.id,
|
jobId: options?.id,
|
||||||
priority: options?.priority,
|
priority: options?.priority,
|
||||||
repeat: {
|
repeat: options?.repeat,
|
||||||
pattern,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|
||||||
await this.queueMap[queueName].add(jobName, data, queueOptions);
|
await this.queueMap[queueName].add(jobName, data, queueOptions);
|
||||||
|
|||||||
@ -3,3 +3,11 @@ export interface QueueJobOptions {
|
|||||||
priority?: number;
|
priority?: number;
|
||||||
retryLimit?: number;
|
retryLimit?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface QueueCronJobOptions extends QueueJobOptions {
|
||||||
|
repeat?: {
|
||||||
|
every?: number;
|
||||||
|
pattern?: string;
|
||||||
|
limit?: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@ -1,4 +1,7 @@
|
|||||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
import {
|
||||||
|
QueueCronJobOptions,
|
||||||
|
QueueJobOptions,
|
||||||
|
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||||
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||||
|
|
||||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||||
@ -18,8 +21,7 @@ export interface MessageQueueDriver {
|
|||||||
queueName: MessageQueue,
|
queueName: MessageQueue,
|
||||||
jobName: string,
|
jobName: string,
|
||||||
data: T,
|
data: T,
|
||||||
pattern: string,
|
options?: QueueCronJobOptions,
|
||||||
options?: QueueJobOptions,
|
|
||||||
);
|
);
|
||||||
removeCron(queueName: MessageQueue, jobName: string, pattern?: string);
|
removeCron(queueName: MessageQueue, jobName: string, pattern?: string);
|
||||||
stop?(): Promise<void>;
|
stop?(): Promise<void>;
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
import PgBoss from 'pg-boss';
|
import PgBoss from 'pg-boss';
|
||||||
|
|
||||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
import {
|
||||||
|
QueueCronJobOptions,
|
||||||
|
QueueJobOptions,
|
||||||
|
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||||
|
|
||||||
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||||
|
|
||||||
@ -8,6 +11,8 @@ import { MessageQueueDriver } from './interfaces/message-queue-driver.interface'
|
|||||||
|
|
||||||
export type PgBossDriverOptions = PgBoss.ConstructorOptions;
|
export type PgBossDriverOptions = PgBoss.ConstructorOptions;
|
||||||
|
|
||||||
|
const DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED = '*/10 * * * *';
|
||||||
|
|
||||||
export class PgBossDriver implements MessageQueueDriver {
|
export class PgBossDriver implements MessageQueueDriver {
|
||||||
private pgBoss: PgBoss;
|
private pgBoss: PgBoss;
|
||||||
|
|
||||||
@ -34,16 +39,15 @@ export class PgBossDriver implements MessageQueueDriver {
|
|||||||
queueName: MessageQueue,
|
queueName: MessageQueue,
|
||||||
jobName: string,
|
jobName: string,
|
||||||
data: T,
|
data: T,
|
||||||
pattern: string,
|
options?: QueueCronJobOptions,
|
||||||
options?: QueueJobOptions,
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await this.pgBoss.schedule(
|
await this.pgBoss.schedule(
|
||||||
`${queueName}.${jobName}`,
|
`${queueName}.${jobName}`,
|
||||||
pattern,
|
options?.repeat?.pattern ??
|
||||||
|
DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED,
|
||||||
data as object,
|
data as object,
|
||||||
options
|
options
|
||||||
? {
|
? {
|
||||||
...options,
|
|
||||||
singletonKey: options?.id,
|
singletonKey: options?.id,
|
||||||
}
|
}
|
||||||
: {},
|
: {},
|
||||||
|
|||||||
@ -33,9 +33,8 @@ export class SyncDriver implements MessageQueueDriver {
|
|||||||
_queueName: MessageQueue,
|
_queueName: MessageQueue,
|
||||||
jobName: string,
|
jobName: string,
|
||||||
data: T,
|
data: T,
|
||||||
pattern: string,
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
this.logger.log(`Running '${pattern}' cron job with SyncDriver`);
|
this.logger.log(`Running cron job with SyncDriver`);
|
||||||
|
|
||||||
const jobClassName = getJobClassName(jobName);
|
const jobClassName = getJobClassName(jobName);
|
||||||
const job: MessageQueueCronJobData<MessageQueueJobData | undefined> =
|
const job: MessageQueueCronJobData<MessageQueueJobData | undefined> =
|
||||||
|
|||||||
@ -44,9 +44,15 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
|
|||||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
import { MessageParticipantObjectMetadata } from 'src/modules/messaging/standard-objects/message-participant.object-metadata';
|
import { MessageParticipantObjectMetadata } from 'src/modules/messaging/standard-objects/message-participant.object-metadata';
|
||||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
|
|
||||||
import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
|
import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
|
||||||
|
import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
|
||||||
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
|
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
|
||||||
|
import { GmailFullSynV2Module } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.module';
|
||||||
|
import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module';
|
||||||
|
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/commands/crons/fetch-all-messages-from-cache.cron-job';
|
||||||
|
import { GmailFullSyncV2Job } from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||||
|
import { GmailPartialSyncV2Job } from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job';
|
||||||
|
import { GmailPartialSyncV2Module } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
@ -78,6 +84,9 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
|
|||||||
MessageChannelObjectMetadata,
|
MessageChannelObjectMetadata,
|
||||||
EventObjectMetadata,
|
EventObjectMetadata,
|
||||||
]),
|
]),
|
||||||
|
GmailFullSynV2Module,
|
||||||
|
GmailFetchMessageContentFromCacheModule,
|
||||||
|
GmailPartialSyncV2Module,
|
||||||
],
|
],
|
||||||
providers: [
|
providers: [
|
||||||
{
|
{
|
||||||
@ -142,6 +151,18 @@ import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.ob
|
|||||||
provide: SaveEventToDbJob.name,
|
provide: SaveEventToDbJob.name,
|
||||||
useClass: SaveEventToDbJob,
|
useClass: SaveEventToDbJob,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
provide: FetchAllMessagesFromCacheCronJob.name,
|
||||||
|
useClass: FetchAllMessagesFromCacheCronJob,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
provide: GmailFullSyncV2Job.name,
|
||||||
|
useClass: GmailFullSyncV2Job,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
provide: GmailPartialSyncV2Job.name,
|
||||||
|
useClass: GmailPartialSyncV2Job,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
export class JobsModule {
|
export class JobsModule {
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
|
import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||||
|
|
||||||
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
import {
|
||||||
|
QueueCronJobOptions,
|
||||||
|
QueueJobOptions,
|
||||||
|
} from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
|
||||||
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
|
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
|
||||||
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||||
|
|
||||||
@ -37,10 +40,9 @@ export class MessageQueueService implements OnModuleDestroy {
|
|||||||
addCron<T extends MessageQueueJobData | undefined>(
|
addCron<T extends MessageQueueJobData | undefined>(
|
||||||
jobName: string,
|
jobName: string,
|
||||||
data: T,
|
data: T,
|
||||||
pattern: string,
|
options?: QueueCronJobOptions,
|
||||||
options?: QueueJobOptions,
|
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
return this.driver.addCron(this.queueName, jobName, data, pattern, options);
|
return this.driver.addCron(this.queueName, jobName, data, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
removeCron(jobName: string, pattern: string): Promise<void> {
|
removeCron(jobName: string, pattern: string): Promise<void> {
|
||||||
|
|||||||
@ -23,8 +23,7 @@ export class StartCleanInactiveWorkspacesCronCommand extends CommandRunner {
|
|||||||
await this.messageQueueService.addCron<undefined>(
|
await this.messageQueueService.addCron<undefined>(
|
||||||
CleanInactiveWorkspaceJob.name,
|
CleanInactiveWorkspaceJob.name,
|
||||||
undefined,
|
undefined,
|
||||||
cleanInactiveWorkspaceCronPattern,
|
{ retryLimit: 3, repeat: { pattern: cleanInactiveWorkspaceCronPattern } },
|
||||||
{ retryLimit: 3 },
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -58,6 +58,7 @@ export class AddStandardIdCommand extends CommandRunner {
|
|||||||
IS_EVENT_OBJECT_ENABLED: true,
|
IS_EVENT_OBJECT_ENABLED: true,
|
||||||
IS_AIRTABLE_INTEGRATION_ENABLED: true,
|
IS_AIRTABLE_INTEGRATION_ENABLED: true,
|
||||||
IS_POSTGRESQL_INTEGRATION_ENABLED: true,
|
IS_POSTGRESQL_INTEGRATION_ENABLED: true,
|
||||||
|
IS_FULL_SYNC_V2_ENABLED: false,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
const standardFieldMetadataCollection = this.standardFieldFactory.create(
|
const standardFieldMetadataCollection = this.standardFieldFactory.create(
|
||||||
@ -72,6 +73,7 @@ export class AddStandardIdCommand extends CommandRunner {
|
|||||||
IS_EVENT_OBJECT_ENABLED: true,
|
IS_EVENT_OBJECT_ENABLED: true,
|
||||||
IS_AIRTABLE_INTEGRATION_ENABLED: true,
|
IS_AIRTABLE_INTEGRATION_ENABLED: true,
|
||||||
IS_POSTGRESQL_INTEGRATION_ENABLED: true,
|
IS_POSTGRESQL_INTEGRATION_ENABLED: true,
|
||||||
|
IS_FULL_SYNC_V2_ENABLED: false,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@ -169,6 +169,10 @@ export const messageChannelStandardFieldIds = {
|
|||||||
type: '20202020-ae95-42d9-a3f1-797a2ea22122',
|
type: '20202020-ae95-42d9-a3f1-797a2ea22122',
|
||||||
isContactAutoCreationEnabled: '20202020-fabd-4f14-b7c6-3310f6d132c6',
|
isContactAutoCreationEnabled: '20202020-fabd-4f14-b7c6-3310f6d132c6',
|
||||||
messageChannelMessageAssociations: '20202020-49b8-4766-88fd-75f1e21b3d5f',
|
messageChannelMessageAssociations: '20202020-49b8-4766-88fd-75f1e21b3d5f',
|
||||||
|
syncExternalId: '20202020-79d1-41cf-b738-bcf5ed61e256',
|
||||||
|
syncedAt: '20202020-263d-4c6b-ad51-137ada56f7d4',
|
||||||
|
syncStatus: '20202020-56a1-4f7e-9880-a8493bb899cc',
|
||||||
|
ongoingSyncStartedAt: '20202020-8c61-4a42-ae63-73c1c3c52e06',
|
||||||
};
|
};
|
||||||
|
|
||||||
export const messageParticipantStandardFieldIds = {
|
export const messageParticipantStandardFieldIds = {
|
||||||
|
|||||||
@ -119,8 +119,9 @@ export class CreateCompanyAndContactService {
|
|||||||
handle: contact.handle,
|
handle: contact.handle,
|
||||||
displayName: contact.displayName,
|
displayName: contact.displayName,
|
||||||
companyId:
|
companyId:
|
||||||
contact.companyDomainName &&
|
contact.companyDomainName && contact.companyDomainName !== ''
|
||||||
companiesObject[contact.companyDomainName],
|
? companiesObject[contact.companyDomainName]
|
||||||
|
: undefined,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
await this.createContactService.createContacts(
|
await this.createContactService.createContacts(
|
||||||
|
|||||||
@ -0,0 +1,84 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import { InjectRepository } from '@nestjs/typeorm';
|
||||||
|
|
||||||
|
import { Repository, In } from 'typeorm';
|
||||||
|
|
||||||
|
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||||
|
|
||||||
|
import {
|
||||||
|
FeatureFlagEntity,
|
||||||
|
FeatureFlagKeys,
|
||||||
|
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||||
|
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||||
|
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||||
|
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||||
|
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||||
|
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
|
import { GmailFetchMessageContentFromCacheService } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class FetchAllMessagesFromCacheCronJob
|
||||||
|
implements MessageQueueJob<undefined>
|
||||||
|
{
|
||||||
|
private readonly logger = new Logger(FetchAllMessagesFromCacheCronJob.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
@InjectRepository(Workspace, 'core')
|
||||||
|
private readonly workspaceRepository: Repository<Workspace>,
|
||||||
|
@InjectRepository(DataSourceEntity, 'metadata')
|
||||||
|
private readonly dataSourceRepository: Repository<DataSourceEntity>,
|
||||||
|
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||||
|
private readonly messageChannelRepository: MessageChannelRepository,
|
||||||
|
@InjectRepository(FeatureFlagEntity, 'core')
|
||||||
|
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||||
|
private readonly gmailFetchMessageContentFromCacheService: GmailFetchMessageContentFromCacheService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async handle(): Promise<void> {
|
||||||
|
const workspaceIds = (
|
||||||
|
await this.workspaceRepository.find({
|
||||||
|
where: {
|
||||||
|
subscriptionStatus: 'active',
|
||||||
|
},
|
||||||
|
select: ['id'],
|
||||||
|
})
|
||||||
|
).map((workspace) => workspace.id);
|
||||||
|
|
||||||
|
const workspacesWithFeatureFlagActive =
|
||||||
|
await this.featureFlagRepository.find({
|
||||||
|
where: {
|
||||||
|
workspaceId: In(workspaceIds),
|
||||||
|
key: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
value: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const dataSources = await this.dataSourceRepository.find({
|
||||||
|
where: {
|
||||||
|
workspaceId: In(
|
||||||
|
workspacesWithFeatureFlagActive.map((w) => w.workspaceId),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const workspaceIdsWithDataSources = new Set(
|
||||||
|
dataSources.map((dataSource) => dataSource.workspaceId),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const workspaceId of workspaceIdsWithDataSources) {
|
||||||
|
await this.fetchWorkspaceMessages(workspaceId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
|
||||||
|
const messageChannels =
|
||||||
|
await this.messageChannelRepository.getAll(workspaceId);
|
||||||
|
|
||||||
|
for (const messageChannel of messageChannels) {
|
||||||
|
await this.gmailFetchMessageContentFromCacheService.fetchMessageContentFromCache(
|
||||||
|
workspaceId,
|
||||||
|
messageChannel.connectedAccountId,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1 +1 @@
|
|||||||
export const fetchAllWorkspacesMessagesCronPattern = '*/10 * * * *';
|
export const fetchAllWorkspacesMessagesCronPattern = '*/5 * * * *';
|
||||||
|
|||||||
@ -16,6 +16,14 @@ import {
|
|||||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
||||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
|
import {
|
||||||
|
FeatureFlagEntity,
|
||||||
|
FeatureFlagKeys,
|
||||||
|
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||||
|
import {
|
||||||
|
GmailPartialSyncV2Job as GmailPartialSyncV2Job,
|
||||||
|
GmailPartialSyncV2JobData as GmailPartialSyncV2JobData,
|
||||||
|
} from 'src/modules/messaging/jobs/gmail-partial-sync-v2.job';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class FetchAllWorkspacesMessagesJob
|
export class FetchAllWorkspacesMessagesJob
|
||||||
@ -32,6 +40,8 @@ export class FetchAllWorkspacesMessagesJob
|
|||||||
private readonly messageQueueService: MessageQueueService,
|
private readonly messageQueueService: MessageQueueService,
|
||||||
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||||
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||||
|
@InjectRepository(FeatureFlagEntity, 'core')
|
||||||
|
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async handle(): Promise<void> {
|
async handle(): Promise<void> {
|
||||||
@ -61,17 +71,33 @@ export class FetchAllWorkspacesMessagesJob
|
|||||||
|
|
||||||
private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
|
private async fetchWorkspaceMessages(workspaceId: string): Promise<void> {
|
||||||
try {
|
try {
|
||||||
|
const isFullSyncV2Enabled = await this.featureFlagRepository.findOneBy({
|
||||||
|
workspaceId,
|
||||||
|
key: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
value: true,
|
||||||
|
});
|
||||||
|
|
||||||
const connectedAccounts =
|
const connectedAccounts =
|
||||||
await this.connectedAccountRepository.getAll(workspaceId);
|
await this.connectedAccountRepository.getAll(workspaceId);
|
||||||
|
|
||||||
for (const connectedAccount of connectedAccounts) {
|
for (const connectedAccount of connectedAccounts) {
|
||||||
await this.messageQueueService.add<GmailPartialSyncJobData>(
|
if (isFullSyncV2Enabled) {
|
||||||
GmailPartialSyncJob.name,
|
await this.messageQueueService.add<GmailPartialSyncV2JobData>(
|
||||||
{
|
GmailPartialSyncV2Job.name,
|
||||||
workspaceId,
|
{
|
||||||
connectedAccountId: connectedAccount.id,
|
workspaceId,
|
||||||
},
|
connectedAccountId: connectedAccount.id,
|
||||||
);
|
},
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
await this.messageQueueService.add<GmailPartialSyncJobData>(
|
||||||
|
GmailPartialSyncJob.name,
|
||||||
|
{
|
||||||
|
workspaceId,
|
||||||
|
connectedAccountId: connectedAccount.id,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
|
|||||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
import { GmailFullSyncCommand } from 'src/modules/messaging/commands/gmail-full-sync.command';
|
import { GmailFullSyncCommand } from 'src/modules/messaging/commands/gmail-full-sync.command';
|
||||||
import { GmailPartialSyncCommand } from 'src/modules/messaging/commands/gmail-partial-sync.command';
|
import { GmailPartialSyncCommand } from 'src/modules/messaging/commands/gmail-partial-sync.command';
|
||||||
|
import { StartFetchAllWorkspacesMessagesFromCacheCronCommand } from 'src/modules/messaging/commands/start-fetch-all-workspaces-messages-from-cache.cron.command';
|
||||||
import { StartFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging/commands/start-fetch-all-workspaces-messages.cron.command';
|
import { StartFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging/commands/start-fetch-all-workspaces-messages.cron.command';
|
||||||
import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging/commands/stop-fetch-all-workspaces-messages.cron.command';
|
import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging/commands/stop-fetch-all-workspaces-messages.cron.command';
|
||||||
|
|
||||||
@ -16,6 +17,7 @@ import { StopFetchAllWorkspacesMessagesCronCommand } from 'src/modules/messaging
|
|||||||
GmailPartialSyncCommand,
|
GmailPartialSyncCommand,
|
||||||
StartFetchAllWorkspacesMessagesCronCommand,
|
StartFetchAllWorkspacesMessagesCronCommand,
|
||||||
StopFetchAllWorkspacesMessagesCronCommand,
|
StopFetchAllWorkspacesMessagesCronCommand,
|
||||||
|
StartFetchAllWorkspacesMessagesFromCacheCronCommand,
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
export class FetchWorkspaceMessagesCommandsModule {}
|
export class FetchWorkspaceMessagesCommandsModule {}
|
||||||
|
|||||||
@ -0,0 +1,32 @@
|
|||||||
|
import { Inject } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { Command, CommandRunner } from 'nest-commander';
|
||||||
|
|
||||||
|
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||||
|
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||||
|
import { FetchAllMessagesFromCacheCronJob } from 'src/modules/messaging/commands/crons/fetch-all-messages-from-cache.cron-job';
|
||||||
|
|
||||||
|
@Command({
|
||||||
|
name: 'fetch-all-workspaces-messages-from-cache:cron:start',
|
||||||
|
description: 'Starts a cron job to fetch all workspaces messages from cache',
|
||||||
|
})
|
||||||
|
export class StartFetchAllWorkspacesMessagesFromCacheCronCommand extends CommandRunner {
|
||||||
|
constructor(
|
||||||
|
@Inject(MessageQueue.cronQueue)
|
||||||
|
private readonly messageQueueService: MessageQueueService,
|
||||||
|
) {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
async run(): Promise<void> {
|
||||||
|
await this.messageQueueService.addCron<undefined>(
|
||||||
|
FetchAllMessagesFromCacheCronJob.name,
|
||||||
|
undefined,
|
||||||
|
{
|
||||||
|
repeat: {
|
||||||
|
every: 5000,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -23,7 +23,9 @@ export class StartFetchAllWorkspacesMessagesCronCommand extends CommandRunner {
|
|||||||
await this.messageQueueService.addCron<undefined>(
|
await this.messageQueueService.addCron<undefined>(
|
||||||
FetchAllWorkspacesMessagesJob.name,
|
FetchAllWorkspacesMessagesJob.name,
|
||||||
undefined,
|
undefined,
|
||||||
fetchAllWorkspacesMessagesCronPattern,
|
{
|
||||||
|
repeat: { pattern: fetchAllWorkspacesMessagesCronPattern },
|
||||||
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1 @@
|
|||||||
|
export const GMAIL_ONGOING_SYNC_TIMEOUT = 1000 * 60 * 60; // 1 hour
|
||||||
@ -0,0 +1 @@
|
|||||||
|
export const GMAIL_USERS_HISTORY_MAX_RESULT = 500;
|
||||||
@ -0,0 +1 @@
|
|||||||
|
export const GMAIL_USERS_MESSAGES_GET_BATCH_SIZE = 50;
|
||||||
@ -0,0 +1 @@
|
|||||||
|
export const GMAIL_USERS_MESSAGES_LIST_MAX_RESULT = 500;
|
||||||
@ -0,0 +1 @@
|
|||||||
|
export const MESSAGES_TO_DELETE_FROM_CACHE_BATCH_SIZE = 1000;
|
||||||
@ -0,0 +1,48 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||||
|
|
||||||
|
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
|
||||||
|
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
|
||||||
|
|
||||||
|
export type GmailFullSyncV2JobData = {
|
||||||
|
workspaceId: string;
|
||||||
|
connectedAccountId: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class GmailFullSyncV2Job
|
||||||
|
implements MessageQueueJob<GmailFullSyncV2JobData>
|
||||||
|
{
|
||||||
|
private readonly logger = new Logger(GmailFullSyncV2Job.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
|
||||||
|
private readonly gmailFullSyncV2Service: GmailFullSyncV2Service,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async handle(data: GmailFullSyncV2JobData): Promise<void> {
|
||||||
|
this.logger.log(
|
||||||
|
`gmail full-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
|
||||||
|
data.workspaceId,
|
||||||
|
data.connectedAccountId,
|
||||||
|
);
|
||||||
|
} catch (e) {
|
||||||
|
this.logger.error(
|
||||||
|
`Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
|
||||||
|
e,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.gmailFullSyncV2Service.fetchConnectedAccountThreads(
|
||||||
|
data.workspaceId,
|
||||||
|
data.connectedAccountId,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -44,7 +44,6 @@ export class GmailFullSyncJob implements MessageQueueJob<GmailFullSyncJobData> {
|
|||||||
await this.gmailFullSyncService.fetchConnectedAccountThreads(
|
await this.gmailFullSyncService.fetchConnectedAccountThreads(
|
||||||
data.workspaceId,
|
data.workspaceId,
|
||||||
data.connectedAccountId,
|
data.connectedAccountId,
|
||||||
data.nextPageToken,
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,48 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
|
||||||
|
|
||||||
|
import { GoogleAPIRefreshAccessTokenService } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.service';
|
||||||
|
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
|
||||||
|
|
||||||
|
export type GmailPartialSyncV2JobData = {
|
||||||
|
workspaceId: string;
|
||||||
|
connectedAccountId: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class GmailPartialSyncV2Job
|
||||||
|
implements MessageQueueJob<GmailPartialSyncV2JobData>
|
||||||
|
{
|
||||||
|
private readonly logger = new Logger(GmailPartialSyncV2Job.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly googleAPIsRefreshAccessTokenService: GoogleAPIRefreshAccessTokenService,
|
||||||
|
private readonly gmailPartialSyncV2Service: GmailPartialSyncV2Service,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async handle(data: GmailPartialSyncV2JobData): Promise<void> {
|
||||||
|
this.logger.log(
|
||||||
|
`gmail partial-sync for workspace ${data.workspaceId} and account ${data.connectedAccountId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.googleAPIsRefreshAccessTokenService.refreshAndSaveAccessToken(
|
||||||
|
data.workspaceId,
|
||||||
|
data.connectedAccountId,
|
||||||
|
);
|
||||||
|
} catch (e) {
|
||||||
|
this.logger.error(
|
||||||
|
`Error refreshing access token for connected account ${data.connectedAccountId} in workspace ${data.workspaceId}`,
|
||||||
|
e,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.gmailPartialSyncV2Service.fetchConnectedAccountThreads(
|
||||||
|
data.workspaceId,
|
||||||
|
data.connectedAccountId,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -3,7 +3,10 @@ import { Injectable } from '@nestjs/common';
|
|||||||
import { EntityManager } from 'typeorm';
|
import { EntityManager } from 'typeorm';
|
||||||
|
|
||||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||||
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
import {
|
||||||
|
MessageChannelObjectMetadata,
|
||||||
|
MessageChannelSyncStatus,
|
||||||
|
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
@ -12,6 +15,21 @@ export class MessageChannelRepository {
|
|||||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
|
public async getAll(
|
||||||
|
workspaceId: string,
|
||||||
|
transactionManager?: EntityManager,
|
||||||
|
): Promise<ObjectRecord<MessageChannelObjectMetadata>[]> {
|
||||||
|
const dataSourceSchema =
|
||||||
|
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||||
|
|
||||||
|
return await this.workspaceDataSourceService.executeRawQuery(
|
||||||
|
`SELECT * FROM ${dataSourceSchema}."messageChannel"`,
|
||||||
|
[],
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
public async getByConnectedAccountId(
|
public async getByConnectedAccountId(
|
||||||
connectedAccountId: string,
|
connectedAccountId: string,
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
@ -49,27 +67,17 @@ export class MessageChannelRepository {
|
|||||||
public async getFirstByConnectedAccountId(
|
public async getFirstByConnectedAccountId(
|
||||||
connectedAccountId: string,
|
connectedAccountId: string,
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
|
transactionManager?: EntityManager,
|
||||||
): Promise<ObjectRecord<MessageChannelObjectMetadata> | undefined> {
|
): Promise<ObjectRecord<MessageChannelObjectMetadata> | undefined> {
|
||||||
const messageChannels = await this.getByConnectedAccountId(
|
const messageChannels = await this.getByConnectedAccountId(
|
||||||
connectedAccountId,
|
connectedAccountId,
|
||||||
workspaceId,
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
);
|
);
|
||||||
|
|
||||||
return messageChannels[0];
|
return messageChannels[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail(
|
|
||||||
connectedAccountId: string,
|
|
||||||
workspaceId: string,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const messageChannel = await this.getFirstByConnectedAccountIdOrFail(
|
|
||||||
connectedAccountId,
|
|
||||||
workspaceId,
|
|
||||||
);
|
|
||||||
|
|
||||||
return messageChannel.isContactAutoCreationEnabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async getByIds(
|
public async getByIds(
|
||||||
ids: string[],
|
ids: string[],
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
@ -85,4 +93,69 @@ export class MessageChannelRepository {
|
|||||||
transactionManager,
|
transactionManager,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async updateSyncStatus(
|
||||||
|
id: string,
|
||||||
|
syncStatus: MessageChannelSyncStatus,
|
||||||
|
workspaceId: string,
|
||||||
|
transactionManager?: EntityManager,
|
||||||
|
): Promise<void> {
|
||||||
|
const dataSourceSchema =
|
||||||
|
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||||
|
|
||||||
|
const needsToUpdateSyncedAt =
|
||||||
|
syncStatus === MessageChannelSyncStatus.SUCCEEDED;
|
||||||
|
|
||||||
|
const needsToUpdateOngoingSyncStartedAt =
|
||||||
|
syncStatus === MessageChannelSyncStatus.ONGOING;
|
||||||
|
|
||||||
|
await this.workspaceDataSourceService.executeRawQuery(
|
||||||
|
`UPDATE ${dataSourceSchema}."messageChannel" SET "syncStatus" = $1 ${
|
||||||
|
needsToUpdateSyncedAt ? `, "syncedAt" = NOW()` : ''
|
||||||
|
} ${
|
||||||
|
needsToUpdateOngoingSyncStartedAt
|
||||||
|
? `, "ongoingSyncStartedAt" = NOW()`
|
||||||
|
: `, "ongoingSyncStartedAt" = NULL`
|
||||||
|
} WHERE "id" = $2`,
|
||||||
|
[syncStatus, id],
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async updateLastSyncExternalIdIfHigher(
|
||||||
|
id: string,
|
||||||
|
syncExternalId: string,
|
||||||
|
workspaceId: string,
|
||||||
|
transactionManager?: EntityManager,
|
||||||
|
) {
|
||||||
|
const dataSourceSchema =
|
||||||
|
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||||
|
|
||||||
|
await this.workspaceDataSourceService.executeRawQuery(
|
||||||
|
`UPDATE ${dataSourceSchema}."messageChannel" SET "syncExternalId" = $1
|
||||||
|
WHERE "id" = $2
|
||||||
|
AND ("syncExternalId" < $1 OR "syncExternalId" = '')`,
|
||||||
|
[syncExternalId, id],
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async resetSyncExternalId(
|
||||||
|
id: string,
|
||||||
|
workspaceId: string,
|
||||||
|
transactionManager?: EntityManager,
|
||||||
|
) {
|
||||||
|
const dataSourceSchema =
|
||||||
|
this.workspaceDataSourceService.getSchemaName(workspaceId);
|
||||||
|
|
||||||
|
await this.workspaceDataSourceService.executeRawQuery(
|
||||||
|
`UPDATE ${dataSourceSchema}."messageChannel" SET "syncExternalId" = ''
|
||||||
|
WHERE "id" = $1`,
|
||||||
|
[id],
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -19,7 +19,6 @@ export class FetchMessagesByBatchesService {
|
|||||||
async fetchAllMessages(
|
async fetchAllMessages(
|
||||||
queries: MessageQuery[],
|
queries: MessageQuery[],
|
||||||
accessToken: string,
|
accessToken: string,
|
||||||
jobName?: string,
|
|
||||||
workspaceId?: string,
|
workspaceId?: string,
|
||||||
connectedAccountId?: string,
|
connectedAccountId?: string,
|
||||||
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
|
): Promise<{ messages: GmailMessage[]; errors: any[] }> {
|
||||||
@ -32,7 +31,7 @@ export class FetchMessagesByBatchesService {
|
|||||||
let endTime = Date.now();
|
let endTime = Date.now();
|
||||||
|
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} fetching ${
|
||||||
queries.length
|
queries.length
|
||||||
} messages in ${endTime - startTime}ms`,
|
} messages in ${endTime - startTime}ms`,
|
||||||
);
|
);
|
||||||
@ -45,7 +44,7 @@ export class FetchMessagesByBatchesService {
|
|||||||
endTime = Date.now();
|
endTime = Date.now();
|
||||||
|
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
`${jobName} for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} formatting ${
|
||||||
queries.length
|
queries.length
|
||||||
} messages in ${endTime - startTime}ms`,
|
} messages in ${endTime - startTime}ms`,
|
||||||
);
|
);
|
||||||
@ -62,6 +61,10 @@ export class FetchMessagesByBatchesService {
|
|||||||
|
|
||||||
const errors: any = [];
|
const errors: any = [];
|
||||||
|
|
||||||
|
const sanitizeString = (str: string) => {
|
||||||
|
return str.replace(/\0/g, '');
|
||||||
|
};
|
||||||
|
|
||||||
const formattedResponse = Promise.all(
|
const formattedResponse = Promise.all(
|
||||||
parsedResponses.map(async (message: GmailMessageParsedResponse) => {
|
parsedResponses.map(async (message: GmailMessageParsedResponse) => {
|
||||||
if (message.error) {
|
if (message.error) {
|
||||||
@ -119,7 +122,7 @@ export class FetchMessagesByBatchesService {
|
|||||||
fromHandle: from.value[0].address || '',
|
fromHandle: from.value[0].address || '',
|
||||||
fromDisplayName: from.value[0].name || '',
|
fromDisplayName: from.value[0].name || '',
|
||||||
participants,
|
participants,
|
||||||
text: textWithoutReplyQuotations || '',
|
text: sanitizeString(textWithoutReplyQuotations || ''),
|
||||||
attachments,
|
attachments,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,26 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||||
|
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||||
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
|
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||||
|
import { GmailFetchMessageContentFromCacheService } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.service';
|
||||||
|
import { MessageModule } from 'src/modules/messaging/services/message/message.module';
|
||||||
|
import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module';
|
||||||
|
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [
|
||||||
|
FetchMessagesByBatchesModule,
|
||||||
|
ObjectMetadataRepositoryModule.forFeature([
|
||||||
|
ConnectedAccountObjectMetadata,
|
||||||
|
MessageChannelObjectMetadata,
|
||||||
|
]),
|
||||||
|
SaveMessageAndEmitContactCreationEventModule,
|
||||||
|
MessageModule,
|
||||||
|
WorkspaceDataSourceModule,
|
||||||
|
],
|
||||||
|
providers: [GmailFetchMessageContentFromCacheService],
|
||||||
|
exports: [GmailFetchMessageContentFromCacheService],
|
||||||
|
})
|
||||||
|
export class GmailFetchMessageContentFromCacheModule {}
|
||||||
@ -0,0 +1,257 @@
|
|||||||
|
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { EntityManager } from 'typeorm';
|
||||||
|
|
||||||
|
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||||
|
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||||
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
|
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||||
|
import { FetchMessagesByBatchesService } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.service';
|
||||||
|
import {
|
||||||
|
MessageChannelObjectMetadata,
|
||||||
|
MessageChannelSyncStatus,
|
||||||
|
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
|
import { createQueriesFromMessageIds } from 'src/modules/messaging/utils/create-queries-from-message-ids.util';
|
||||||
|
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||||
|
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||||
|
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||||
|
import { GMAIL_USERS_MESSAGES_GET_BATCH_SIZE } from 'src/modules/messaging/constants/gmail-users-messages-get-batch-size.constant';
|
||||||
|
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||||
|
import { SaveMessageAndEmitContactCreationEventService } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.service';
|
||||||
|
import {
|
||||||
|
GmailFullSyncV2JobData,
|
||||||
|
GmailFullSyncV2Job,
|
||||||
|
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||||
|
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||||
|
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||||
|
import { GMAIL_ONGOING_SYNC_TIMEOUT } from 'src/modules/messaging/constants/gmail-ongoing-sync-timeout.constant';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class GmailFetchMessageContentFromCacheService {
|
||||||
|
private readonly logger = new Logger(
|
||||||
|
GmailFetchMessageContentFromCacheService.name,
|
||||||
|
);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly fetchMessagesByBatchesService: FetchMessagesByBatchesService,
|
||||||
|
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||||
|
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||||
|
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||||
|
private readonly messageChannelRepository: MessageChannelRepository,
|
||||||
|
private readonly saveMessageAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||||
|
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||||
|
private readonly cacheStorage: CacheStorageService,
|
||||||
|
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||||
|
@Inject(MessageQueue.messagingQueue)
|
||||||
|
private readonly messageQueueService: MessageQueueService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async fetchMessageContentFromCache(
|
||||||
|
workspaceId: string,
|
||||||
|
connectedAccountId: string,
|
||||||
|
) {
|
||||||
|
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||||
|
connectedAccountId,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!connectedAccount) {
|
||||||
|
this.logger.error(
|
||||||
|
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const accessToken = connectedAccount.accessToken;
|
||||||
|
const refreshToken = connectedAccount.refreshToken;
|
||||||
|
|
||||||
|
if (!refreshToken) {
|
||||||
|
throw new Error(
|
||||||
|
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const gmailMessageChannel =
|
||||||
|
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||||
|
connectedAccountId,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!gmailMessageChannel) {
|
||||||
|
this.logger.error(
|
||||||
|
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.PENDING) {
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is not pending.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.ONGOING) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ongoingSyncStartedAt = new Date(
|
||||||
|
gmailMessageChannel.ongoingSyncStartedAt,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (
|
||||||
|
ongoingSyncStartedAt < new Date(Date.now() - GMAIL_ONGOING_SYNC_TIMEOUT)
|
||||||
|
) {
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} failed due to ongoing sync timeout. Restarting full-sync...`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.FAILED,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const gmailMessageChannelId = gmailMessageChannel.id;
|
||||||
|
|
||||||
|
const messageIdsToFetch =
|
||||||
|
(await this.cacheStorage.setPop(
|
||||||
|
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannelId}`,
|
||||||
|
GMAIL_USERS_MESSAGES_GET_BATCH_SIZE,
|
||||||
|
)) ?? [];
|
||||||
|
|
||||||
|
if (!messageIdsToFetch?.length) {
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannelId,
|
||||||
|
MessageChannelSyncStatus.SUCCEEDED,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import or delete.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannelId,
|
||||||
|
MessageChannelSyncStatus.ONGOING,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} starting...`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const workspaceDataSource =
|
||||||
|
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
await workspaceDataSource
|
||||||
|
?.transaction(async (transactionManager: EntityManager) => {
|
||||||
|
const messageQueries = createQueriesFromMessageIds(messageIdsToFetch);
|
||||||
|
|
||||||
|
const { messages: messagesToSave, errors } =
|
||||||
|
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||||
|
messageQueries,
|
||||||
|
accessToken,
|
||||||
|
workspaceId,
|
||||||
|
connectedAccountId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!messagesToSave.length) {
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannelId,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errors.length) {
|
||||||
|
const errorsCanBeIgnored = errors.every(
|
||||||
|
(error) => error.code === 404,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!errorsCanBeIgnored) {
|
||||||
|
throw new Error(
|
||||||
|
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${JSON.stringify(
|
||||||
|
errors,
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.saveMessageAndEmitContactCreationEventService.saveMessagesAndEmitContactCreationEventWithinTransaction(
|
||||||
|
messagesToSave,
|
||||||
|
connectedAccount,
|
||||||
|
workspaceId,
|
||||||
|
gmailMessageChannel,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (messageIdsToFetch.length < GMAIL_USERS_MESSAGES_GET_BATCH_SIZE) {
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannelId,
|
||||||
|
MessageChannelSyncStatus.SUCCEEDED,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with no more messages to import.`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannelId,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} done with more messages to import.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(async (error) => {
|
||||||
|
await this.cacheStorage.setAdd(
|
||||||
|
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannelId}`,
|
||||||
|
messageIdsToFetch,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannelId,
|
||||||
|
MessageChannelSyncStatus.FAILED,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async fallbackToFullSync(
|
||||||
|
workspaceId: string,
|
||||||
|
connectedAccountId: string,
|
||||||
|
) {
|
||||||
|
await this.messageQueueService.add<GmailFullSyncV2JobData>(
|
||||||
|
GmailFullSyncV2Job.name,
|
||||||
|
{ workspaceId, connectedAccountId },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,31 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||||
|
|
||||||
|
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||||
|
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||||
|
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||||
|
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||||
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
|
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||||
|
import { GmailFullSyncV2Service } from 'src/modules/messaging/services/gmail-full-sync-v2/gmail-full-sync.v2.service';
|
||||||
|
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||||
|
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||||
|
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [
|
||||||
|
MessagingProvidersModule,
|
||||||
|
FetchMessagesByBatchesModule,
|
||||||
|
ObjectMetadataRepositoryModule.forFeature([
|
||||||
|
ConnectedAccountObjectMetadata,
|
||||||
|
MessageChannelObjectMetadata,
|
||||||
|
MessageChannelMessageAssociationObjectMetadata,
|
||||||
|
BlocklistObjectMetadata,
|
||||||
|
]),
|
||||||
|
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||||
|
WorkspaceDataSourceModule,
|
||||||
|
],
|
||||||
|
providers: [GmailFullSyncV2Service],
|
||||||
|
exports: [GmailFullSyncV2Service],
|
||||||
|
})
|
||||||
|
export class GmailFullSynV2Module {}
|
||||||
@ -0,0 +1,302 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import { InjectRepository } from '@nestjs/typeorm';
|
||||||
|
|
||||||
|
import { EntityManager, Repository } from 'typeorm';
|
||||||
|
import { gmail_v1 } from 'googleapis';
|
||||||
|
|
||||||
|
import {
|
||||||
|
FeatureFlagEntity,
|
||||||
|
FeatureFlagKeys,
|
||||||
|
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||||
|
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||||
|
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||||
|
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||||
|
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||||
|
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
|
||||||
|
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||||
|
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||||
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
|
import { GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/constants/gmail-users-messages-list-max-result.constant';
|
||||||
|
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/repositories/message-channel-message-association.repository';
|
||||||
|
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||||
|
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||||
|
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||||
|
import {
|
||||||
|
MessageChannelObjectMetadata,
|
||||||
|
MessageChannelSyncStatus,
|
||||||
|
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
|
import { gmailSearchFilterExcludeEmails } from 'src/modules/messaging/utils/gmail-search-filter.util';
|
||||||
|
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class GmailFullSyncV2Service {
|
||||||
|
private readonly logger = new Logger(GmailFullSyncV2Service.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly gmailClientProvider: GmailClientProvider,
|
||||||
|
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||||
|
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||||
|
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||||
|
private readonly messageChannelRepository: MessageChannelRepository,
|
||||||
|
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||||
|
private readonly blocklistRepository: BlocklistRepository,
|
||||||
|
@InjectRepository(FeatureFlagEntity, 'core')
|
||||||
|
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||||
|
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||||
|
private readonly cacheStorage: CacheStorageService,
|
||||||
|
@InjectObjectMetadataRepository(
|
||||||
|
MessageChannelMessageAssociationObjectMetadata,
|
||||||
|
)
|
||||||
|
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||||
|
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public async fetchConnectedAccountThreads(
|
||||||
|
workspaceId: string,
|
||||||
|
connectedAccountId: string,
|
||||||
|
) {
|
||||||
|
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||||
|
connectedAccountId,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!connectedAccount) {
|
||||||
|
this.logger.error(
|
||||||
|
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const refreshToken = connectedAccount.refreshToken;
|
||||||
|
|
||||||
|
if (!refreshToken) {
|
||||||
|
throw new Error(
|
||||||
|
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const gmailMessageChannel =
|
||||||
|
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||||
|
connectedAccountId,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!gmailMessageChannel) {
|
||||||
|
this.logger.error(
|
||||||
|
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (gmailMessageChannel.syncStatus === MessageChannelSyncStatus.ONGOING) {
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is locked, import will be retried later.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.ONGOING,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
const workspaceDataSource =
|
||||||
|
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
await workspaceDataSource
|
||||||
|
?.transaction(async (transactionManager) => {
|
||||||
|
const gmailClient: gmail_v1.Gmail =
|
||||||
|
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||||
|
|
||||||
|
const blocklistedEmails = await this.fetchBlocklistEmails(
|
||||||
|
connectedAccount.accountOwnerId,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||||
|
gmailClient,
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
blocklistedEmails,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.catch(async (error) => {
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.FAILED,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public async fetchAllMessageIdsFromGmailAndStoreInCache(
|
||||||
|
gmailClient: gmail_v1.Gmail,
|
||||||
|
messageChannelId: string,
|
||||||
|
blocklistedEmails: string[],
|
||||||
|
workspaceId: string,
|
||||||
|
transactionManager?: EntityManager,
|
||||||
|
) {
|
||||||
|
let pageToken: string | undefined;
|
||||||
|
let hasMoreMessages = true;
|
||||||
|
let messageIdsToFetch = 0;
|
||||||
|
let firstMessageExternalId;
|
||||||
|
|
||||||
|
while (hasMoreMessages) {
|
||||||
|
const response = await gmailClient.users.messages.list({
|
||||||
|
userId: 'me',
|
||||||
|
maxResults: GMAIL_USERS_MESSAGES_LIST_MAX_RESULT,
|
||||||
|
pageToken,
|
||||||
|
q: gmailSearchFilterExcludeEmails(blocklistedEmails),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.data?.messages) {
|
||||||
|
const messageExternalIds = response.data.messages
|
||||||
|
.filter((message): message is { id: string } => message.id != null)
|
||||||
|
.map((message) => message.id);
|
||||||
|
|
||||||
|
if (!firstMessageExternalId) {
|
||||||
|
firstMessageExternalId = messageExternalIds[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
const existingMessageChannelMessageAssociations =
|
||||||
|
await this.messageChannelMessageAssociationRepository.getByMessageExternalIdsAndMessageChannelId(
|
||||||
|
messageExternalIds,
|
||||||
|
messageChannelId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
const existingMessageChannelMessageAssociationsExternalIds =
|
||||||
|
existingMessageChannelMessageAssociations.map(
|
||||||
|
(messageChannelMessageAssociation) =>
|
||||||
|
messageChannelMessageAssociation.messageExternalId,
|
||||||
|
);
|
||||||
|
|
||||||
|
const messageIdsToImport = messageExternalIds.filter(
|
||||||
|
(messageExternalId) =>
|
||||||
|
!existingMessageChannelMessageAssociationsExternalIds.includes(
|
||||||
|
messageExternalId,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (messageIdsToImport && messageIdsToImport.length) {
|
||||||
|
await this.cacheStorage.setAdd(
|
||||||
|
`messages-to-import:${workspaceId}:gmail:${messageChannelId}`,
|
||||||
|
messageIdsToImport,
|
||||||
|
);
|
||||||
|
|
||||||
|
messageIdsToFetch += messageIdsToImport.length;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pageToken = response.data.nextPageToken ?? undefined;
|
||||||
|
hasMoreMessages = !!pageToken;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!messageIdsToFetch) {
|
||||||
|
this.logger.log(
|
||||||
|
`No messages found in Gmail for messageChannel ${messageChannelId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`Fetched all ${messageIdsToFetch} message ids from Gmail for messageChannel ${messageChannelId} in workspace ${workspaceId} and added to cache for import`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.updateLastSyncExternalId(
|
||||||
|
gmailClient,
|
||||||
|
messageChannelId,
|
||||||
|
firstMessageExternalId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async fetchBlocklistEmails(
|
||||||
|
workspaceMemberId: string,
|
||||||
|
workspaceId: string,
|
||||||
|
) {
|
||||||
|
const isBlocklistEnabledFeatureFlag =
|
||||||
|
await this.featureFlagRepository.findOneBy({
|
||||||
|
workspaceId,
|
||||||
|
key: FeatureFlagKeys.IsBlocklistEnabled,
|
||||||
|
value: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const isBlocklistEnabled =
|
||||||
|
isBlocklistEnabledFeatureFlag && isBlocklistEnabledFeatureFlag.value;
|
||||||
|
|
||||||
|
const blocklist = isBlocklistEnabled
|
||||||
|
? await this.blocklistRepository.getByWorkspaceMemberId(
|
||||||
|
workspaceMemberId,
|
||||||
|
workspaceId,
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
return blocklist.map((blocklist) => blocklist.handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async updateLastSyncExternalId(
|
||||||
|
gmailClient: gmail_v1.Gmail,
|
||||||
|
messageChannelId: string,
|
||||||
|
firstMessageExternalId: string,
|
||||||
|
workspaceId: string,
|
||||||
|
transactionManager?: EntityManager,
|
||||||
|
) {
|
||||||
|
if (!firstMessageExternalId) {
|
||||||
|
throw new Error(
|
||||||
|
`No first message found for workspace ${workspaceId} and account ${messageChannelId}, can't update sync external id`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstMessageContent = await gmailClient.users.messages.get({
|
||||||
|
userId: 'me',
|
||||||
|
id: firstMessageExternalId,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!firstMessageContent?.data) {
|
||||||
|
throw new Error(
|
||||||
|
`No first message content found for message ${firstMessageExternalId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const historyId = firstMessageContent?.data?.historyId;
|
||||||
|
|
||||||
|
if (!historyId) {
|
||||||
|
throw new Error(
|
||||||
|
`No historyId found for message ${firstMessageExternalId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.log(
|
||||||
|
`Updating last external id: ${historyId} for workspace ${workspaceId} and account ${messageChannelId} succeeded.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateLastSyncExternalIdIfHigher(
|
||||||
|
messageChannelId,
|
||||||
|
historyId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -47,7 +47,7 @@ export class GmailFullSyncService {
|
|||||||
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
private readonly messageChannelMessageAssociationRepository: MessageChannelMessageAssociationRepository,
|
||||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||||
private readonly blocklistRepository: BlocklistRepository,
|
private readonly blocklistRepository: BlocklistRepository,
|
||||||
private readonly saveMessagesAndCreateContactsService: SaveMessageAndEmitContactCreationEventService,
|
private readonly saveMessagesAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||||
@InjectRepository(FeatureFlagEntity, 'core')
|
@InjectRepository(FeatureFlagEntity, 'core')
|
||||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||||
) {}
|
) {}
|
||||||
@ -186,7 +186,6 @@ export class GmailFullSyncService {
|
|||||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||||
messageQueries,
|
messageQueries,
|
||||||
accessToken,
|
accessToken,
|
||||||
'gmail full-sync',
|
|
||||||
workspaceId,
|
workspaceId,
|
||||||
connectedAccountId,
|
connectedAccountId,
|
||||||
);
|
);
|
||||||
@ -200,12 +199,11 @@ export class GmailFullSyncService {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if (messagesToSave.length > 0) {
|
if (messagesToSave.length > 0) {
|
||||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
await this.saveMessagesAndEmitContactCreationEventService.saveMessagesAndEmitContactCreation(
|
||||||
messagesToSave,
|
messagesToSave,
|
||||||
connectedAccount,
|
connectedAccount,
|
||||||
workspaceId,
|
workspaceId,
|
||||||
gmailMessageChannelId,
|
gmailMessageChannelId,
|
||||||
'gmail full-sync',
|
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
|
|||||||
@ -0,0 +1,33 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||||
|
|
||||||
|
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||||
|
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
|
||||||
|
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||||
|
import { BlocklistObjectMetadata } from 'src/modules/connected-account/standard-objects/blocklist.object-metadata';
|
||||||
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
|
import { FetchMessagesByBatchesModule } from 'src/modules/messaging/services/fetch-messages-by-batches/fetch-messages-by-batches.module';
|
||||||
|
import { GmailPartialSyncV2Service } from 'src/modules/messaging/services/gmail-partial-sync-v2/gmail-partial-sync-v2.service';
|
||||||
|
import { MessageModule } from 'src/modules/messaging/services/message/message.module';
|
||||||
|
import { MessagingProvidersModule } from 'src/modules/messaging/services/providers/messaging-providers.module';
|
||||||
|
import { SaveMessageAndEmitContactCreationEventModule } from 'src/modules/messaging/services/save-message-and-emit-contact-creation-event/save-message-and-emit-contact-creation-event.module';
|
||||||
|
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [
|
||||||
|
MessagingProvidersModule,
|
||||||
|
FetchMessagesByBatchesModule,
|
||||||
|
ObjectMetadataRepositoryModule.forFeature([
|
||||||
|
ConnectedAccountObjectMetadata,
|
||||||
|
MessageChannelObjectMetadata,
|
||||||
|
BlocklistObjectMetadata,
|
||||||
|
]),
|
||||||
|
MessageModule,
|
||||||
|
SaveMessageAndEmitContactCreationEventModule,
|
||||||
|
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
|
||||||
|
WorkspaceDataSourceModule,
|
||||||
|
],
|
||||||
|
providers: [GmailPartialSyncV2Service],
|
||||||
|
exports: [GmailPartialSyncV2Service],
|
||||||
|
})
|
||||||
|
export class GmailPartialSyncV2Module {}
|
||||||
@ -0,0 +1,338 @@
|
|||||||
|
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { gmail_v1 } from 'googleapis';
|
||||||
|
import { EntityManager } from 'typeorm';
|
||||||
|
|
||||||
|
import { GmailClientProvider } from 'src/modules/messaging/services/providers/gmail/gmail-client.provider';
|
||||||
|
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
|
||||||
|
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
|
||||||
|
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
|
||||||
|
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||||
|
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||||
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
|
import {
|
||||||
|
MessageChannelObjectMetadata,
|
||||||
|
MessageChannelSyncStatus,
|
||||||
|
} from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
|
||||||
|
import { GMAIL_USERS_HISTORY_MAX_RESULT } from 'src/modules/messaging/constants/gmail-users-history-max-result.constant';
|
||||||
|
import { GmailError } from 'src/modules/messaging/types/gmail-error';
|
||||||
|
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
|
||||||
|
import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator';
|
||||||
|
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
|
||||||
|
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||||
|
import {
|
||||||
|
GmailFullSyncV2Job,
|
||||||
|
GmailFullSyncV2JobData,
|
||||||
|
} from 'src/modules/messaging/jobs/gmail-full-sync-v2.job';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class GmailPartialSyncV2Service {
|
||||||
|
private readonly logger = new Logger(GmailPartialSyncV2Service.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly gmailClientProvider: GmailClientProvider,
|
||||||
|
@Inject(MessageQueue.messagingQueue)
|
||||||
|
private readonly messageQueueService: MessageQueueService,
|
||||||
|
@InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
|
||||||
|
private readonly connectedAccountRepository: ConnectedAccountRepository,
|
||||||
|
@InjectObjectMetadataRepository(MessageChannelObjectMetadata)
|
||||||
|
private readonly messageChannelRepository: MessageChannelRepository,
|
||||||
|
@InjectCacheStorage(CacheStorageNamespace.Messaging)
|
||||||
|
private readonly cacheStorage: CacheStorageService,
|
||||||
|
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public async fetchConnectedAccountThreads(
|
||||||
|
workspaceId: string,
|
||||||
|
connectedAccountId: string,
|
||||||
|
): Promise<void> {
|
||||||
|
const connectedAccount = await this.connectedAccountRepository.getById(
|
||||||
|
connectedAccountId,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!connectedAccount) {
|
||||||
|
this.logger.error(
|
||||||
|
`Connected account ${connectedAccountId} not found in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const refreshToken = connectedAccount.refreshToken;
|
||||||
|
|
||||||
|
if (!refreshToken) {
|
||||||
|
throw new Error(
|
||||||
|
`No refresh token found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const gmailMessageChannel =
|
||||||
|
await this.messageChannelRepository.getFirstByConnectedAccountId(
|
||||||
|
connectedAccountId,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!gmailMessageChannel) {
|
||||||
|
this.logger.error(
|
||||||
|
`No message channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (gmailMessageChannel.syncStatus !== MessageChannelSyncStatus.SUCCEEDED) {
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import for workspace ${workspaceId} and account ${connectedAccountId} is locked, import will be retried later.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.ONGOING,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
const workspaceDataSource =
|
||||||
|
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
await workspaceDataSource
|
||||||
|
?.transaction(async (transactionManager: EntityManager) => {
|
||||||
|
const lastSyncHistoryId = gmailMessageChannel.syncExternalId;
|
||||||
|
|
||||||
|
if (!lastSyncHistoryId) {
|
||||||
|
this.logger.log(
|
||||||
|
`No lastSyncHistoryId for workspace ${workspaceId} and account ${connectedAccountId}, falling back to full sync.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const gmailClient: gmail_v1.Gmail =
|
||||||
|
await this.gmailClientProvider.getGmailClient(refreshToken);
|
||||||
|
|
||||||
|
const { history, historyId, error } = await this.getHistoryFromGmail(
|
||||||
|
gmailClient,
|
||||||
|
lastSyncHistoryId,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (error?.code === 404) {
|
||||||
|
this.logger.log(
|
||||||
|
`404: Invalid lastSyncHistoryId: ${lastSyncHistoryId} for workspace ${workspaceId} and account ${connectedAccountId}, falling back to full sync.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.resetSyncExternalId(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.fallbackToFullSync(workspaceId, connectedAccountId);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (error?.code === 429) {
|
||||||
|
this.logger.log(
|
||||||
|
`429: rate limit reached for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}, import will be retried later.`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
throw new Error(
|
||||||
|
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!historyId) {
|
||||||
|
throw new Error(
|
||||||
|
`No historyId found for ${connectedAccountId} in workspace ${workspaceId} in gmail history response.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (historyId === lastSyncHistoryId || !history?.length) {
|
||||||
|
this.logger.log(
|
||||||
|
`Messaging import done with history ${historyId} and nothing to update for workspace ${workspaceId} and account ${connectedAccountId}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { messagesAdded, messagesDeleted } =
|
||||||
|
await this.getMessageIdsFromHistory(history);
|
||||||
|
|
||||||
|
await this.cacheStorage.setAdd(
|
||||||
|
`messages-to-import:${workspaceId}:gmail:${gmailMessageChannel.id}`,
|
||||||
|
messagesAdded,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.cacheStorage.setAdd(
|
||||||
|
`messages-to-delete:${workspaceId}:gmail:${gmailMessageChannel.id}`,
|
||||||
|
messagesDeleted,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateLastSyncExternalIdIfHigher(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
historyId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.PENDING,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.catch(async (error) => {
|
||||||
|
await this.messageChannelRepository.updateSyncStatus(
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
MessageChannelSyncStatus.FAILED,
|
||||||
|
workspaceId,
|
||||||
|
);
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`Error fetching messages for ${connectedAccountId} in workspace ${workspaceId}: ${error.message}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getMessageIdsFromHistory(
|
||||||
|
history: gmail_v1.Schema$History[],
|
||||||
|
): Promise<{
|
||||||
|
messagesAdded: string[];
|
||||||
|
messagesDeleted: string[];
|
||||||
|
}> {
|
||||||
|
const { messagesAdded, messagesDeleted } = history.reduce(
|
||||||
|
(
|
||||||
|
acc: {
|
||||||
|
messagesAdded: string[];
|
||||||
|
messagesDeleted: string[];
|
||||||
|
},
|
||||||
|
history,
|
||||||
|
) => {
|
||||||
|
const messagesAdded = history.messagesAdded?.map(
|
||||||
|
(messageAdded) => messageAdded.message?.id || '',
|
||||||
|
);
|
||||||
|
|
||||||
|
const messagesDeleted = history.messagesDeleted?.map(
|
||||||
|
(messageDeleted) => messageDeleted.message?.id || '',
|
||||||
|
);
|
||||||
|
|
||||||
|
if (messagesAdded) acc.messagesAdded.push(...messagesAdded);
|
||||||
|
if (messagesDeleted) acc.messagesDeleted.push(...messagesDeleted);
|
||||||
|
|
||||||
|
return acc;
|
||||||
|
},
|
||||||
|
{ messagesAdded: [], messagesDeleted: [] },
|
||||||
|
);
|
||||||
|
|
||||||
|
const uniqueMessagesAdded = messagesAdded.filter(
|
||||||
|
(messageId) => !messagesDeleted.includes(messageId),
|
||||||
|
);
|
||||||
|
|
||||||
|
const uniqueMessagesDeleted = messagesDeleted.filter(
|
||||||
|
(messageId) => !messagesAdded.includes(messageId),
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
messagesAdded: uniqueMessagesAdded,
|
||||||
|
messagesDeleted: uniqueMessagesDeleted,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getHistoryFromGmail(
|
||||||
|
gmailClient: gmail_v1.Gmail,
|
||||||
|
lastSyncHistoryId: string,
|
||||||
|
): Promise<{
|
||||||
|
history: gmail_v1.Schema$History[];
|
||||||
|
historyId?: string | null;
|
||||||
|
error?: GmailError;
|
||||||
|
}> {
|
||||||
|
const fullHistory: gmail_v1.Schema$History[] = [];
|
||||||
|
let pageToken: string | undefined;
|
||||||
|
let hasMoreMessages = true;
|
||||||
|
let nextHistoryId: string | undefined;
|
||||||
|
|
||||||
|
while (hasMoreMessages) {
|
||||||
|
try {
|
||||||
|
const response = await gmailClient.users.history.list({
|
||||||
|
userId: 'me',
|
||||||
|
maxResults: GMAIL_USERS_HISTORY_MAX_RESULT,
|
||||||
|
pageToken,
|
||||||
|
startHistoryId: lastSyncHistoryId,
|
||||||
|
historyTypes: ['messageAdded', 'messageDeleted'],
|
||||||
|
});
|
||||||
|
|
||||||
|
nextHistoryId = response?.data?.historyId ?? undefined;
|
||||||
|
|
||||||
|
if (response?.data?.history) {
|
||||||
|
fullHistory.push(...response.data.history);
|
||||||
|
}
|
||||||
|
|
||||||
|
pageToken = response?.data?.nextPageToken ?? undefined;
|
||||||
|
hasMoreMessages = !!pageToken;
|
||||||
|
} catch (error) {
|
||||||
|
const errorData = error?.response?.data?.error;
|
||||||
|
|
||||||
|
if (errorData) {
|
||||||
|
return {
|
||||||
|
history: [],
|
||||||
|
error: errorData,
|
||||||
|
historyId: lastSyncHistoryId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { history: fullHistory, historyId: nextHistoryId };
|
||||||
|
}
|
||||||
|
|
||||||
|
private async fallbackToFullSync(
|
||||||
|
workspaceId: string,
|
||||||
|
connectedAccountId: string,
|
||||||
|
) {
|
||||||
|
await this.messageQueueService.add<GmailFullSyncV2JobData>(
|
||||||
|
GmailFullSyncV2Job.name,
|
||||||
|
{ workspaceId, connectedAccountId },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -45,7 +45,7 @@ export class GmailPartialSyncService {
|
|||||||
private readonly messageService: MessageService,
|
private readonly messageService: MessageService,
|
||||||
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
@InjectObjectMetadataRepository(BlocklistObjectMetadata)
|
||||||
private readonly blocklistRepository: BlocklistRepository,
|
private readonly blocklistRepository: BlocklistRepository,
|
||||||
private readonly saveMessagesAndCreateContactsService: SaveMessageAndEmitContactCreationEventService,
|
private readonly saveMessagesAndEmitContactCreationEventService: SaveMessageAndEmitContactCreationEventService,
|
||||||
@InjectRepository(FeatureFlagEntity, 'core')
|
@InjectRepository(FeatureFlagEntity, 'core')
|
||||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||||
) {}
|
) {}
|
||||||
@ -174,7 +174,6 @@ export class GmailPartialSyncService {
|
|||||||
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
await this.fetchMessagesByBatchesService.fetchAllMessages(
|
||||||
messageQueries,
|
messageQueries,
|
||||||
accessToken,
|
accessToken,
|
||||||
'gmail partial-sync',
|
|
||||||
workspaceId,
|
workspaceId,
|
||||||
connectedAccountId,
|
connectedAccountId,
|
||||||
);
|
);
|
||||||
@ -208,12 +207,11 @@ export class GmailPartialSyncService {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if (messagesToSave.length !== 0) {
|
if (messagesToSave.length !== 0) {
|
||||||
await this.saveMessagesAndCreateContactsService.saveMessagesAndCreateContacts(
|
await this.saveMessagesAndEmitContactCreationEventService.saveMessagesAndEmitContactCreation(
|
||||||
messagesToSave,
|
messagesToSave,
|
||||||
connectedAccount,
|
connectedAccount,
|
||||||
workspaceId,
|
workspaceId,
|
||||||
gmailMessageChannelId,
|
gmailMessageChannelId,
|
||||||
'gmail partial-sync',
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -6,7 +6,6 @@ import { v4 } from 'uuid';
|
|||||||
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
|
||||||
import { MessageObjectMetadata } from 'src/modules/messaging/standard-objects/message.object-metadata';
|
import { MessageObjectMetadata } from 'src/modules/messaging/standard-objects/message.object-metadata';
|
||||||
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
|
||||||
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
|
|
||||||
import { GmailMessage } from 'src/modules/messaging/types/gmail-message';
|
import { GmailMessage } from 'src/modules/messaging/types/gmail-message';
|
||||||
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
|
||||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
@ -38,9 +37,67 @@ export class MessageService {
|
|||||||
private readonly messageThreadService: MessageThreadService,
|
private readonly messageThreadService: MessageThreadService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
|
public async saveMessagesWithinTransaction(
|
||||||
|
messages: GmailMessage[],
|
||||||
|
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||||
|
gmailMessageChannelId: string,
|
||||||
|
workspaceId: string,
|
||||||
|
transactionManager: EntityManager,
|
||||||
|
): Promise<Map<string, string>> {
|
||||||
|
const messageExternalIdsAndIdsMap = new Map<string, string>();
|
||||||
|
|
||||||
|
for (const message of messages) {
|
||||||
|
const existingMessageChannelMessageAssociationsCount =
|
||||||
|
await this.messageChannelMessageAssociationRepository.countByMessageExternalIdsAndMessageChannelId(
|
||||||
|
[message.externalId],
|
||||||
|
gmailMessageChannelId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (existingMessageChannelMessageAssociationsCount > 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: This does not handle all thread merging use cases and might create orphan threads.
|
||||||
|
const savedOrExistingMessageThreadId =
|
||||||
|
await this.messageThreadService.saveMessageThreadOrReturnExistingMessageThread(
|
||||||
|
message.headerMessageId,
|
||||||
|
message.messageThreadExternalId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
const savedOrExistingMessageId =
|
||||||
|
await this.saveMessageOrReturnExistingMessage(
|
||||||
|
message,
|
||||||
|
savedOrExistingMessageThreadId,
|
||||||
|
connectedAccount,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
messageExternalIdsAndIdsMap.set(
|
||||||
|
message.externalId,
|
||||||
|
savedOrExistingMessageId,
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.messageChannelMessageAssociationRepository.insert(
|
||||||
|
gmailMessageChannelId,
|
||||||
|
savedOrExistingMessageId,
|
||||||
|
message.externalId,
|
||||||
|
savedOrExistingMessageThreadId,
|
||||||
|
message.messageThreadExternalId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return messageExternalIdsAndIdsMap;
|
||||||
|
}
|
||||||
|
|
||||||
public async saveMessages(
|
public async saveMessages(
|
||||||
messages: GmailMessage[],
|
messages: GmailMessage[],
|
||||||
dataSourceMetadata: DataSourceEntity,
|
|
||||||
workspaceDataSource: DataSource,
|
workspaceDataSource: DataSource,
|
||||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||||
gmailMessageChannelId: string,
|
gmailMessageChannelId: string,
|
||||||
@ -101,7 +158,6 @@ export class MessageService {
|
|||||||
message,
|
message,
|
||||||
savedOrExistingMessageThreadId,
|
savedOrExistingMessageThreadId,
|
||||||
connectedAccount,
|
connectedAccount,
|
||||||
dataSourceMetadata,
|
|
||||||
workspaceId,
|
workspaceId,
|
||||||
manager,
|
manager,
|
||||||
);
|
);
|
||||||
@ -136,7 +192,6 @@ export class MessageService {
|
|||||||
message: GmailMessage,
|
message: GmailMessage,
|
||||||
messageThreadId: string,
|
messageThreadId: string,
|
||||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||||
dataSourceMetadata: DataSourceEntity,
|
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
manager: EntityManager,
|
manager: EntityManager,
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
|
|||||||
@ -1,6 +1,8 @@
|
|||||||
import { Injectable, Logger } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||||
|
|
||||||
|
import { EntityManager } from 'typeorm';
|
||||||
|
|
||||||
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
|
||||||
import { MessageParticipantRepository } from 'src/modules/messaging/repositories/message-participant.repository';
|
import { MessageParticipantRepository } from 'src/modules/messaging/repositories/message-participant.repository';
|
||||||
import {
|
import {
|
||||||
@ -31,14 +33,65 @@ export class SaveMessageAndEmitContactCreationEventService {
|
|||||||
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async saveMessagesAndCreateContacts(
|
public async saveMessagesAndEmitContactCreationEventWithinTransaction(
|
||||||
|
messagesToSave: GmailMessage[],
|
||||||
|
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||||
|
workspaceId: string,
|
||||||
|
gmailMessageChannel: ObjectRecord<MessageChannelObjectMetadata>,
|
||||||
|
transactionManager: EntityManager,
|
||||||
|
) {
|
||||||
|
const messageExternalIdsAndIdsMap =
|
||||||
|
await this.messageService.saveMessagesWithinTransaction(
|
||||||
|
messagesToSave,
|
||||||
|
connectedAccount,
|
||||||
|
gmailMessageChannel.id,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
const participantsWithMessageId: (ParticipantWithMessageId & {
|
||||||
|
shouldCreateContact: boolean;
|
||||||
|
})[] = messagesToSave.flatMap((message) => {
|
||||||
|
const messageId = messageExternalIdsAndIdsMap.get(message.externalId);
|
||||||
|
|
||||||
|
return messageId
|
||||||
|
? message.participants.map((participant) => ({
|
||||||
|
...participant,
|
||||||
|
messageId,
|
||||||
|
shouldCreateContact:
|
||||||
|
gmailMessageChannel.isContactAutoCreationEnabled &&
|
||||||
|
message.participants.find((p) => p.role === 'from')?.handle ===
|
||||||
|
connectedAccount.handle,
|
||||||
|
}))
|
||||||
|
: [];
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.messageParticipantRepository.saveMessageParticipants(
|
||||||
|
participantsWithMessageId,
|
||||||
|
workspaceId,
|
||||||
|
transactionManager,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (gmailMessageChannel.isContactAutoCreationEnabled) {
|
||||||
|
const contactsToCreate = participantsWithMessageId.filter(
|
||||||
|
(participant) => participant.shouldCreateContact,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.eventEmitter.emit(`createContacts`, {
|
||||||
|
workspaceId,
|
||||||
|
connectedAccountHandle: connectedAccount.handle,
|
||||||
|
contactsToCreate,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async saveMessagesAndEmitContactCreation(
|
||||||
messagesToSave: GmailMessage[],
|
messagesToSave: GmailMessage[],
|
||||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
gmailMessageChannelId: string,
|
gmailMessageChannelId: string,
|
||||||
jobName?: string,
|
|
||||||
) {
|
) {
|
||||||
const { dataSource: workspaceDataSource, dataSourceMetadata } =
|
const { dataSource: workspaceDataSource } =
|
||||||
await this.workspaceDataSourceService.connectedToWorkspaceDataSourceAndReturnMetadata(
|
await this.workspaceDataSourceService.connectedToWorkspaceDataSourceAndReturnMetadata(
|
||||||
workspaceId,
|
workspaceId,
|
||||||
);
|
);
|
||||||
@ -47,7 +100,6 @@ export class SaveMessageAndEmitContactCreationEventService {
|
|||||||
|
|
||||||
const messageExternalIdsAndIdsMap = await this.messageService.saveMessages(
|
const messageExternalIdsAndIdsMap = await this.messageService.saveMessages(
|
||||||
messagesToSave,
|
messagesToSave,
|
||||||
dataSourceMetadata,
|
|
||||||
workspaceDataSource,
|
workspaceDataSource,
|
||||||
connectedAccount,
|
connectedAccount,
|
||||||
gmailMessageChannelId,
|
gmailMessageChannelId,
|
||||||
@ -57,7 +109,7 @@ export class SaveMessageAndEmitContactCreationEventService {
|
|||||||
let endTime = Date.now();
|
let endTime = Date.now();
|
||||||
|
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
`${jobName} saving messages for workspace ${workspaceId} and account ${
|
`Saving messages for workspace ${workspaceId} and account ${
|
||||||
connectedAccount.id
|
connectedAccount.id
|
||||||
} in ${endTime - startTime}ms`,
|
} in ${endTime - startTime}ms`,
|
||||||
);
|
);
|
||||||
@ -100,13 +152,12 @@ export class SaveMessageAndEmitContactCreationEventService {
|
|||||||
gmailMessageChannel,
|
gmailMessageChannel,
|
||||||
workspaceId,
|
workspaceId,
|
||||||
connectedAccount,
|
connectedAccount,
|
||||||
jobName,
|
|
||||||
);
|
);
|
||||||
|
|
||||||
endTime = Date.now();
|
endTime = Date.now();
|
||||||
|
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
`${jobName} saving message participants for workspace ${workspaceId} and account in ${
|
`Saving message participants for workspace ${workspaceId} and account in ${
|
||||||
connectedAccount.id
|
connectedAccount.id
|
||||||
} ${endTime - startTime}ms`,
|
} ${endTime - startTime}ms`,
|
||||||
);
|
);
|
||||||
@ -119,7 +170,6 @@ export class SaveMessageAndEmitContactCreationEventService {
|
|||||||
gmailMessageChannel: ObjectRecord<MessageChannelObjectMetadata>,
|
gmailMessageChannel: ObjectRecord<MessageChannelObjectMetadata>,
|
||||||
workspaceId: string,
|
workspaceId: string,
|
||||||
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
connectedAccount: ObjectRecord<ConnectedAccountObjectMetadata>,
|
||||||
jobName?: string,
|
|
||||||
) {
|
) {
|
||||||
try {
|
try {
|
||||||
await this.messageParticipantRepository.saveMessageParticipants(
|
await this.messageParticipantRepository.saveMessageParticipants(
|
||||||
@ -140,7 +190,7 @@ export class SaveMessageAndEmitContactCreationEventService {
|
|||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
`${jobName} error saving message participants for workspace ${workspaceId} and account ${connectedAccount.id}`,
|
`Error saving message participants for workspace ${workspaceId} and account ${connectedAccount.id}`,
|
||||||
error,
|
error,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@ -1,3 +1,4 @@
|
|||||||
|
import { FeatureFlagKeys } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
|
||||||
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||||
import {
|
import {
|
||||||
RelationMetadataType,
|
RelationMetadataType,
|
||||||
@ -6,6 +7,7 @@ import {
|
|||||||
import { messageChannelStandardFieldIds } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
|
import { messageChannelStandardFieldIds } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
|
||||||
import { standardObjectIds } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
|
import { standardObjectIds } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
|
||||||
import { FieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/field-metadata.decorator';
|
import { FieldMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/field-metadata.decorator';
|
||||||
|
import { Gate } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/gate.decorator';
|
||||||
import { IsNullable } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/is-nullable.decorator';
|
import { IsNullable } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/is-nullable.decorator';
|
||||||
import { IsSystem } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/is-system.decorator';
|
import { IsSystem } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/is-system.decorator';
|
||||||
import { ObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/object-metadata.decorator';
|
import { ObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/decorators/object-metadata.decorator';
|
||||||
@ -14,6 +16,13 @@ import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-
|
|||||||
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
|
||||||
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
|
||||||
|
|
||||||
|
export enum MessageChannelSyncStatus {
|
||||||
|
PENDING = 'PENDING',
|
||||||
|
ONGOING = 'ONGOING',
|
||||||
|
SUCCEEDED = 'SUCCEEDED',
|
||||||
|
FAILED = 'FAILED',
|
||||||
|
}
|
||||||
|
|
||||||
@ObjectMetadata({
|
@ObjectMetadata({
|
||||||
standardId: standardObjectIds.messageChannel,
|
standardId: standardObjectIds.messageChannel,
|
||||||
namePlural: 'messageChannels',
|
namePlural: 'messageChannels',
|
||||||
@ -102,4 +111,81 @@ export class MessageChannelObjectMetadata extends BaseObjectMetadata {
|
|||||||
})
|
})
|
||||||
@IsNullable()
|
@IsNullable()
|
||||||
messageChannelMessageAssociations: MessageChannelMessageAssociationObjectMetadata[];
|
messageChannelMessageAssociations: MessageChannelMessageAssociationObjectMetadata[];
|
||||||
|
|
||||||
|
@FieldMetadata({
|
||||||
|
standardId: messageChannelStandardFieldIds.syncExternalId,
|
||||||
|
type: FieldMetadataType.TEXT,
|
||||||
|
label: 'Last sync external ID',
|
||||||
|
description: 'Last sync external ID',
|
||||||
|
icon: 'IconHistory',
|
||||||
|
})
|
||||||
|
@Gate({
|
||||||
|
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
})
|
||||||
|
syncExternalId: string;
|
||||||
|
|
||||||
|
@FieldMetadata({
|
||||||
|
standardId: messageChannelStandardFieldIds.syncedAt,
|
||||||
|
type: FieldMetadataType.DATE_TIME,
|
||||||
|
label: 'Last sync date',
|
||||||
|
description: 'Last sync date',
|
||||||
|
icon: 'IconHistory',
|
||||||
|
})
|
||||||
|
@Gate({
|
||||||
|
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
})
|
||||||
|
@IsNullable()
|
||||||
|
syncedAt: string;
|
||||||
|
|
||||||
|
@FieldMetadata({
|
||||||
|
standardId: messageChannelStandardFieldIds.syncStatus,
|
||||||
|
type: FieldMetadataType.SELECT,
|
||||||
|
label: 'Last sync status',
|
||||||
|
description: 'Last sync status',
|
||||||
|
icon: 'IconHistory',
|
||||||
|
options: [
|
||||||
|
{
|
||||||
|
value: MessageChannelSyncStatus.PENDING,
|
||||||
|
label: 'Pending',
|
||||||
|
position: 0,
|
||||||
|
color: 'blue',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
value: MessageChannelSyncStatus.ONGOING,
|
||||||
|
label: 'Ongoing',
|
||||||
|
position: 1,
|
||||||
|
color: 'yellow',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
value: MessageChannelSyncStatus.SUCCEEDED,
|
||||||
|
label: 'Succeeded',
|
||||||
|
position: 2,
|
||||||
|
color: 'green',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
value: MessageChannelSyncStatus.FAILED,
|
||||||
|
label: 'Failed',
|
||||||
|
position: 3,
|
||||||
|
color: 'red',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
@Gate({
|
||||||
|
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
})
|
||||||
|
@IsNullable()
|
||||||
|
syncStatus: MessageChannelSyncStatus;
|
||||||
|
|
||||||
|
@FieldMetadata({
|
||||||
|
standardId: messageChannelStandardFieldIds.ongoingSyncStartedAt,
|
||||||
|
type: FieldMetadataType.DATE_TIME,
|
||||||
|
label: 'Ongoing sync started at',
|
||||||
|
description: 'Ongoing sync started at',
|
||||||
|
icon: 'IconHistory',
|
||||||
|
})
|
||||||
|
@Gate({
|
||||||
|
featureFlag: FeatureFlagKeys.IsFullSyncV2Enabled,
|
||||||
|
})
|
||||||
|
@IsNullable()
|
||||||
|
ongoingSyncStartedAt: string;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,11 @@
|
|||||||
|
export type GmailError = {
|
||||||
|
code: number;
|
||||||
|
errors: {
|
||||||
|
domain: string;
|
||||||
|
reason: string;
|
||||||
|
message: string;
|
||||||
|
locationType?: string;
|
||||||
|
location?: string;
|
||||||
|
}[];
|
||||||
|
message: string;
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user