admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Optimize sync, reset, seed commands to flush cache and to use less memory (#7034)

Browse Source 
In this PR:
- removing ugprade-0.24 commands as we are releasing 0.30
- introducing cache:flush command
- refactoring upgrade command and sync-metadata command to use the
ActiveWorkspacesCommand so they consistently run on all workspaces or
selected workspaces

Fixes:
- clear localStorage on sign out
- fix missing workspaceMember in verify resolver
- do not throw on datasource already destroyed exception which can
happen with race condition when several resolvers are resolving in
parallel
This commit is contained in:
Charles Bochet
2024-09-15 12:47:45 +02:00
committed by GitHub
parent 0dbd4a7665
commit f54eea0227
31 changed files with 110 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts
View File

@ -30,11 +30,11 @@ import { seedPeople } from 'src/database/typeorm-seeds/workspace/people';
import { seedWorkspaceMember } from 'src/database/typeorm-seeds/workspace/workspace-members';
import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/cache-storage.service';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';

6
packages/twenty-server/src/database/commands/database-command.module.ts
View File

@ -7,7 +7,6 @@ import { DataSeedDemoWorkspaceCommand } from 'src/database/commands/data-seed-de
import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module';
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
import { UpgradeTo0_24CommandModule } from 'src/database/commands/upgrade-version/0-24/0-24-upgrade-version.module';
import { UpgradeTo0_30CommandModule } from 'src/database/commands/upgrade-version/0-30/0-30-upgrade-version.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
@ -20,9 +19,9 @@ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module';
@Module({
@ -41,12 +40,11 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
WorkspaceModule,
WorkspaceDataSourceModule,
WorkspaceSyncMetadataModule,
WorkspaceStatusModule,
ObjectMetadataModule,
FieldMetadataModule,
DataSeedDemoWorkspaceModule,
WorkspaceCacheStorageModule,
WorkspaceMetadataVersionModule,
UpgradeTo0_24CommandModule,
UpgradeTo0_30CommandModule,
],
providers: [

222
packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command.ts
View File

@ -1,222 +0,0 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import chalk from 'chalk';
import { Command, CommandRunner, Option } from 'nest-commander';
import { Any, Repository } from 'typeorm';
import {
Workspace,
WorkspaceActivationStatus,
} from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { MessageDirection } from 'src/modules/messaging/common/enums/message-direction.enum';
import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
interface SetMessageDirectionCommandOptions {
workspaceId?: string;
}
const MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE = 10;
@Command({
name: 'upgrade-0.24:set-message-direction',
description: 'Set message direction',
})
export class SetMessageDirectionCommand extends CommandRunner {
private readonly logger = new Logger(SetMessageDirectionCommand.name);
constructor(
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
) {
super();
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id. Command runs on all workspaces if not provided',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
async run(
_passedParam: string[],
options: SetMessageDirectionCommandOptions,
): Promise<void> {
let activeWorkspaceIds: string[] = [];
if (options.workspaceId) {
activeWorkspaceIds = [options.workspaceId];
} else {
const activeWorkspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
...(options.workspaceId && { id: options.workspaceId }),
},
});
activeWorkspaceIds = activeWorkspaces.map((workspace) => workspace.id);
}
if (!activeWorkspaceIds.length) {
this.logger.log(chalk.yellow('No workspace found'));
return;
} else {
this.logger.log(
chalk.green(
`Running command on ${activeWorkspaceIds.length} workspaces`,
),
);
}
for (const workspaceId of activeWorkspaceIds) {
try {
const messageChannelMessageAssociationRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const workspaceDataSource =
await this.twentyORMGlobalManager.getDataSourceForWorkspace(
workspaceId,
);
await workspaceDataSource.transaction(async (transactionManager) => {
try {
const messageChannelMessageAssociationCount =
await messageChannelMessageAssociationRepository.count(
{},
transactionManager,
);
for (
let i = 0;
i < messageChannelMessageAssociationCount;
i += MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE
) {
const messageChannelMessageAssociationsPage =
await messageChannelMessageAssociationRepository.find(
{
where: {
message: {
messageParticipants: {
role: 'from',
},
},
},
relations: {
message: {
messageParticipants: true,
},
messageChannel: {
connectedAccount: true,
},
},
take: MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE,
skip: i,
},
transactionManager,
);
const { incoming, outgoing } =
messageChannelMessageAssociationsPage.reduce(
(
acc: {
incoming: string[];
outgoing: string[];
},
messageChannelMessageAssociation,
) => {
const connectedAccountHandle =
messageChannelMessageAssociation?.messageChannel
?.connectedAccount?.handle;
const connectedAccountHandleAliases =
messageChannelMessageAssociation?.messageChannel
?.connectedAccount?.handleAliases;
const fromHandle =
messageChannelMessageAssociation?.message
?.messageParticipants?.[0]?.handle ?? '';
if (
connectedAccountHandle === fromHandle ||
connectedAccountHandleAliases?.includes(fromHandle)
) {
acc.outgoing.push(messageChannelMessageAssociation.id);
} else {
acc.incoming.push(messageChannelMessageAssociation.id);
}
return acc;
},
{ incoming: [], outgoing: [] },
);
await messageChannelMessageAssociationRepository.update(
{
id: Any(incoming),
},
{
direction: MessageDirection.INCOMING,
},
transactionManager,
);
await messageChannelMessageAssociationRepository.update(
{
id: Any(outgoing),
},
{
direction: MessageDirection.OUTGOING,
},
transactionManager,
);
const numberOfProcessedAssociations =
i + MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE;
if (
numberOfProcessedAssociations %
(MESSAGE_CHANNEL_MESSAGE_ASSOCIATION_BATCH_SIZE * 10) ===
0 ||
numberOfProcessedAssociations >=
messageChannelMessageAssociationCount
) {
this.logger.log(
chalk.green(
`Processed ${Math.min(numberOfProcessedAssociations, messageChannelMessageAssociationCount)} of ${messageChannelMessageAssociationCount} message channel message associations`,
),
);
}
}
} catch (error) {
this.logger.log(
chalk.red(`Running command on workspace ${workspaceId} failed`),
);
throw error;
}
});
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
this.logger.log(
chalk.green(`Running command on workspace ${workspaceId} done`),
);
} catch (error) {
this.logger.error(
`Migration failed for workspace ${workspaceId}: ${error.message}`,
);
}
}
this.logger.log(chalk.green(`Command completed!`));
}
}

42
packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-upgrade-version.command.ts
View File

@ -1,42 +0,0 @@
import { Command, CommandRunner, Option } from 'nest-commander';
import { SetMessageDirectionCommand } from 'src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command';
import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
interface UpdateTo0_24CommandOptions {
workspaceId?: string;
}
@Command({
name: 'upgrade-0.24',
description: 'Upgrade to 0.24',
})
export class UpgradeTo0_24Command extends CommandRunner {
constructor(
private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
private readonly setMessagesDirectionCommand: SetMessageDirectionCommand,
) {
super();
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description:
'workspace id. Command runs on all active workspaces if not provided',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
async run(
passedParam: string[],
options: UpdateTo0_24CommandOptions,
): Promise<void> {
await this.syncWorkspaceMetadataCommand.run(passedParam, {
...options,
force: true,
});
await this.setMessagesDirectionCommand.run(passedParam, options);
}
}

43
packages/twenty-server/src/database/commands/upgrade-version/0-24/0-24-upgrade-version.module.ts
View File

@ -1,43 +0,0 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { SetMessageDirectionCommand } from 'src/database/commands/upgrade-version/0-24/0-24-set-message-direction.command';
import { UpgradeTo0_24Command } from 'src/database/commands/upgrade-version/0-24/0-24-upgrade-version.command';
import { SetCustomObjectIsSoftDeletableCommand } from 'src/database/commands/upgrade-version/0-30/0-30-set-custom-object-is-soft-deletable.command';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity';
import { OnboardingModule } from 'src/engine/core-modules/onboarding/onboarding.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { FileStorageModule } from 'src/engine/core-modules/file-storage/file-storage.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace, KeyValuePair], 'core'),
WorkspaceSyncMetadataCommandsModule,
FileStorageModule,
OnboardingModule,
TypeORMModule,
DataSourceModule,
WorkspaceMetadataVersionModule,
FieldMetadataModule,
WorkspaceStatusModule,
TypeOrmModule.forFeature(
[FieldMetadataEntity, ObjectMetadataEntity],
'metadata',
),
TypeORMModule,
],
providers: [
UpgradeTo0_24Command,
SetMessageDirectionCommand,
SetCustomObjectIsSoftDeletableCommand,
],
})
export class UpgradeTo0_24CommandModule {}

50
packages/twenty-server/src/database/commands/upgrade-version/0-30/0-30-upgrade-version.command.ts
View File

@ -1,7 +1,12 @@
import { Command, CommandRunner, Option } from 'nest-commander';
import { InjectRepository } from '@nestjs/typeorm';
import { Command } from 'nest-commander';
import { Repository } from 'typeorm';
import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
import { MigrateEmailFieldsToEmailsCommand } from 'src/database/commands/upgrade-version/0-30/0-30-migrate-email-fields-to-emails.command';
import { SetCustomObjectIsSoftDeletableCommand } from 'src/database/commands/upgrade-version/0-30/0-30-set-custom-object-is-soft-deletable.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
interface UpdateTo0_30CommandOptions {
@ -12,34 +17,39 @@ interface UpdateTo0_30CommandOptions {
name: 'upgrade-0.30',
description: 'Upgrade to 0.30',
})
export class UpgradeTo0_30Command extends CommandRunner {
export class UpgradeTo0_30Command extends ActiveWorkspacesCommandRunner {
constructor(
@InjectRepository(Workspace, 'core')
protected readonly workspaceRepository: Repository<Workspace>,
private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
private readonly migrateEmailFieldsToEmails: MigrateEmailFieldsToEmailsCommand,
private readonly setCustomObjectIsSoftDeletableCommand: SetCustomObjectIsSoftDeletableCommand,
) {
super();
super(workspaceRepository);
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description:
'workspace id. Command runs on all active workspaces if not provided',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
async run(
async executeActiveWorkspacesCommand(
passedParam: string[],
options: UpdateTo0_30CommandOptions,
workspaceIds: string[],
): Promise<void> {
await this.syncWorkspaceMetadataCommand.run(passedParam, {
...options,
force: true,
});
await this.setCustomObjectIsSoftDeletableCommand.run(passedParam, options);
await this.migrateEmailFieldsToEmails.run(passedParam, options);
await this.syncWorkspaceMetadataCommand.executeActiveWorkspacesCommand(
passedParam,
{
...options,
force: true,
},
workspaceIds,
);
await this.setCustomObjectIsSoftDeletableCommand.executeActiveWorkspacesCommand(
passedParam,
options,
workspaceIds,
);
await this.migrateEmailFieldsToEmails.executeActiveWorkspacesCommand(
passedParam,
options,
workspaceIds,
);
}
}