admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance twenty orm (#6691)

Browse Source 
## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts
View File

@ -1,7 +1,7 @@
import { Module } from '@nestjs/common';
import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
@ -11,7 +11,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor
WorkspaceMigrationModule,
WorkspaceMigrationRunnerModule,
WorkspaceDataSourceModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
],
providers: [ForeignTableService],
exports: [ForeignTableService],

8
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts
View File

@ -11,7 +11,7 @@ import {
} from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.exception';
import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
ReferencedTable,
@ -30,7 +30,7 @@ export class ForeignTableService {
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
) {}
public async fetchForeignTableNamesWithinWorkspace(
@ -127,7 +127,9 @@ export class ForeignTableService {
workspaceId,
);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return {
name: foreignTableName,

4
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts
View File

@ -11,7 +11,7 @@ import { RemoteTableSchemaUpdateModule } from 'src/engine/metadata-modules/remot
import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
@ -24,7 +24,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
DataSourceModule,
ObjectMetadataModule,
FieldMetadataModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
WorkspaceDataSourceModule,
ForeignTableModule,
RemoteTableSchemaUpdateModule,

66
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts
View File

@ -1,49 +1,49 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { plural } from 'pluralize';
import isEmpty from 'lodash.isempty';
import { plural } from 'pluralize';
import { Repository } from 'typeorm';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import {
RemoteServerType,
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service';
import { sortDistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import {
DistantTableUpdate,
RemoteTableStatus,
} from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import {
mapUdtNameToFieldSettings,
mapUdtNameToFieldType,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { camelCase } from 'src/utils/camel-case';
import { camelToTitleCase } from 'src/utils/camel-to-title-case';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util';
import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util';
import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service';
import { RemoteTableSchemaUpdateService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service';
import { sortDistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
import {
RemoteTableException,
RemoteTableExceptionCode,
} from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.exception';
import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util';
import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util';
import {
mapUdtNameToFieldSettings,
mapUdtNameToFieldType,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { camelCase } from 'src/utils/camel-case';
import { camelToTitleCase } from 'src/utils/camel-to-title-case';
export class RemoteTableService {
private readonly logger = new Logger(RemoteTableService.name);
@ -55,7 +55,7 @@ export class RemoteTableService {
private readonly remoteServerRepository: Repository<
RemoteServerEntity<RemoteServerType>
>,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly dataSourceService: DataSourceService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly fieldMetadataService: FieldMetadataService,
@ -250,7 +250,9 @@ export class RemoteTableService {
await this.remoteTableRepository.save(remoteTableEntity);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return {
id: remoteTableEntity.id,
@ -435,7 +437,9 @@ export class RemoteTableService {
await this.remoteTableRepository.delete(remoteTable.id);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
}
private async createRemoteTableMetadata(