admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance twenty orm (#6691)

Browse Source 
## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts
View File

@ -1,8 +1,7 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Command, CommandRunner } from 'nest-commander';
import { EntityManager, Repository } from 'typeorm';
import { EntityManager } from 'typeorm';
import { seedCoreSchema } from 'src/database/typeorm-seeds/core';
import {
@ -39,7 +38,6 @@ import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { viewPrefillData } from 'src/engine/workspace-manager/standard-objects-prefill-data/view';
import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
@ -61,12 +59,9 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly fieldMetadataService: FieldMetadataService,
@InjectRepository(ObjectMetadataEntity, 'metadata')
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly objectMetadataService: ObjectMetadataService,
@InjectCacheStorage(CacheStorageNamespace.WorkspaceSchema)
@InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
private readonly workspaceSchemaCache: CacheStorageService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
) {
super();
}
@ -75,7 +70,6 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
try {
for (const workspaceId of this.workspaceIds) {
await this.workspaceSchemaCache.flush();
await this.workspaceCacheVersionService.deleteVersion(workspaceId);
await rawDataSource.initialize();

4
packages/twenty-server/src/database/commands/database-command.module.ts
View File

@ -19,7 +19,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
@ -45,7 +45,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
ObjectMetadataModule,
FieldMetadataModule,
DataSeedDemoWorkspaceModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
UpgradeTo0_23CommandModule,
UpgradeVersionModule,
],

8
packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts
View File

@ -10,7 +10,7 @@ import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service';
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
@ -34,7 +34,7 @@ export class MigrateMessageChannelSyncStatusEnumCommand extends CommandRunner {
private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
private readonly typeORMService: TypeORMService,
private readonly dataSourceService: DataSourceService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
) {
super();
}
@ -212,7 +212,9 @@ export class MigrateMessageChannelSyncStatusEnumCommand extends CommandRunner {
options: newOptions,
});
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
this.logger.log(
chalk.green(`Running command on workspace ${workspaceId} done`),

8
packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts
View File

@ -13,7 +13,7 @@ import {
Workspace,
WorkspaceActivationStatus,
} from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
@ -34,7 +34,7 @@ export class SetUserVarsAccountsToReconnectCommand extends CommandRunner {
SetUserVarsAccountsToReconnectCommand.name,
);
constructor(
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
private readonly accountsToReconnectService: AccountsToReconnectService,
@InjectRepository(KeyValuePair, 'core')
@ -149,7 +149,9 @@ export class SetUserVarsAccountsToReconnectCommand extends CommandRunner {
throw error;
}
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
this.logger.log(
chalk.green(`Running command on workspace ${workspaceId} done`),

8
packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts
View File

@ -12,7 +12,7 @@ import {
WorkspaceActivationStatus,
} from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
interface SetWorkspaceActivationStatusCommandOptions {
workspaceId?: string;
@ -31,7 +31,7 @@ export class SetWorkspaceActivationStatusCommand extends CommandRunner {
private readonly workspaceRepository: Repository<Workspace>,
private readonly typeORMService: TypeORMService,
private readonly dataSourceService: DataSourceService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly billingSubscriptionService: BillingSubscriptionService,
) {
super();
@ -97,7 +97,9 @@ export class SetWorkspaceActivationStatusCommand extends CommandRunner {
}
}
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
this.logger.log(
chalk.green(`Running command on workspace ${workspaceId} done`),

8
packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts
View File

@ -9,7 +9,7 @@ import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
import { notesAllView } from 'src/engine/workspace-manager/standard-objects-prefill-data/views/notes-all.view';
import { tasksAllView } from 'src/engine/workspace-manager/standard-objects-prefill-data/views/tasks-all.view';
@ -44,7 +44,7 @@ export class UpdateActivitiesCommand extends CommandRunner {
private readonly workspaceStatusService: WorkspaceStatusService,
private readonly typeORMService: TypeORMService,
private readonly dataSourceService: DataSourceService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {
@ -400,7 +400,9 @@ export class UpdateActivitiesCommand extends CommandRunner {
.execute();
}
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
}
}

4
packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts
View File

@ -21,7 +21,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
@ -35,7 +35,7 @@ import { ViewModule } from 'src/modules/view/view.module';
OnboardingModule,
TypeORMModule,
DataSourceModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
FieldMetadataModule,
DataSourceModule,
WorkspaceStatusModule,

4
packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts
View File

@ -21,7 +21,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
@ -48,7 +48,7 @@ import { ViewModule } from 'src/modules/view/view.module';
WorkspaceStatusModule,
ObjectMetadataModule,
DataSeedDemoWorkspaceModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
FieldMetadataModule,
ViewModule,
BillingModule,

31
packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts Normal file
View File

@ -0,0 +1,31 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class IntroduceMetadataVersionAndDatasourceOnWorkspace1724173430043
implements MigrationInterface
{
name = 'IntroduceMetadataVersionAndDatasourceOnWorkspace1724173430043';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."workspace" ADD "metadataVersion" integer NOT NULL DEFAULT '1'`,
);
await queryRunner.query(
`ALTER TABLE "core"."workspace" ADD "databaseUrl" character varying NOT NULL DEFAULT ''`,
);
await queryRunner.query(
`ALTER TABLE "core"."workspace" ADD "databaseSchema" character varying NOT NULL DEFAULT ''`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "core"."workspace" DROP COLUMN "databaseSchema"`,
);
await queryRunner.query(
`ALTER TABLE "core"."workspace" DROP COLUMN "databaseUrl"`,
);
await queryRunner.query(
`ALTER TABLE "core"."workspace" DROP COLUMN "metadataVersion"`,
);
}
}

15
packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts Normal file
View File

@ -0,0 +1,15 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class DeprecateCacheVersion1724173061204 implements MigrationInterface {
name = 'DeprecateCacheVersion1724173061204';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "metadata"."workspaceCacheVersion"`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "metadata"."workspaceCacheVersion" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "workspaceId" uuid NOT NULL, "version" character varying NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_1a80ecf2638b477809403cc26ed" UNIQUE ("workspaceId"), CONSTRAINT "PK_5d502f8dbfb5b9a8bf2439320e9" PRIMARY KEY ("id"))`,
);
}
}