[messaging] remove partial sync retry and fix missing datasource error (#4371)

* [messaging] remove partial sync retry and fix missing datasource error

* revert

* fix

* add 429

* fix

* fix

* fix

* remove duplicate log

* fix cron pattern
This commit is contained in:
Weiko
2024-03-08 14:06:21 +01:00
committed by GitHub
parent d2e2e50d8a
commit 250bb6134e
5 changed files with 35 additions and 12 deletions

View File

@ -1,7 +1,7 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Repository, In } from 'typeorm';
import { MessageQueueJob } from 'src/integrations/message-queue/interfaces/message-queue-job.interface';
@ -13,6 +13,7 @@ import {
GmailPartialSyncJobData,
GmailPartialSyncJob,
} from 'src/workspace/messaging/jobs/gmail-partial-sync.job';
import { DataSourceEntity } from 'src/metadata/data-source/data-source.entity';
@Injectable()
export class FetchAllWorkspacesMessagesJob
@ -23,6 +24,8 @@ export class FetchAllWorkspacesMessagesJob
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
@InjectRepository(DataSourceEntity, 'metadata')
private readonly dataSourceRepository: Repository<DataSourceEntity>,
@Inject(MessageQueue.messagingQueue)
private readonly messageQueueService: MessageQueueService,
private readonly connectedAccountService: ConnectedAccountService,
@ -38,7 +41,17 @@ export class FetchAllWorkspacesMessagesJob
})
).map((workspace) => workspace.id);
for (const workspaceId of workspaceIds) {
const dataSources = await this.dataSourceRepository.find({
where: {
workspaceId: In(workspaceIds),
},
});
const workspaceIdsWithDataSources = new Set(
dataSources.map((dataSource) => dataSource.workspaceId),
);
for (const workspaceId of workspaceIdsWithDataSources) {
await this.fetchWorkspaceMessages(workspaceId);
}
}
@ -55,9 +68,6 @@ export class FetchAllWorkspacesMessagesJob
workspaceId,
connectedAccountId: connectedAccount.id,
},
{
retryLimit: 2,
},
);
}
} catch (error) {

View File

@ -56,9 +56,6 @@ export class GmailPartialSyncCommand extends CommandRunner {
workspaceId,
connectedAccountId: connectedAccount.id,
},
{
retryLimit: 2,
},
);
}
}