admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance twenty orm (#6691)

Browse Source 
## Context

As we grow, the messaging scripts are experiencing performance issues
forcing us to temporarily disable them on the cloud.
While investigating the performance, I have noticed that generating the
entity schema (for twentyORM) in the repository is taking ~500ms locally
on my Mac M2 so likely more on pods. Caching the entitySchema then!

I'm also clarifying naming around schemaVersion and cacheVersions ==>
both are renamed workspaceMetadataVersion and migrated to the workspace
table (the workspaceCacheVersion table is dropped).
This commit is contained in:
Charles Bochet
2024-08-20 19:42:02 +02:00
committed by GitHub
parent 3ae89d15de
commit 17a1760afd
80 changed files with 583 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts
View File

@ -13,15 +13,15 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s
import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
import { FieldMetadataResolver } from 'src/engine/metadata-modules/field-metadata/field-metadata.resolver';
import { FieldMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/field-metadata/interceptors/field-metadata-graphql-api-exception.interceptor';
import { CreatedByPreQueryHook } from 'src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook';
import { IsFieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-default-value.validator';
import { IsFieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-options.validator';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module';
import { CreatedByPreQueryHook } from 'src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook';
import { FieldMetadataEntity } from './field-metadata.entity';
import { FieldMetadataService } from './field-metadata.service';
@ -38,7 +38,7 @@ import { UpdateFieldInput } from './dtos/update-field.input';
WorkspaceStatusModule,
TwentyORMModule,
WorkspaceMigrationRunnerModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
ObjectMetadataModule,
DataSourceModule,
TypeORMModule,

16
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
View File

@ -40,7 +40,7 @@ import {
NameTooLongException,
validateMetadataNameOrThrow,
} from 'src/engine/metadata-modules/utils/validate-metadata-name.utils';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,
@ -76,7 +76,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly dataSourceService: DataSourceService,
private readonly typeORMService: TypeORMService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
) {
super(fieldMetadataRepository);
}
@ -274,7 +274,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
throw error;
} finally {
await queryRunner.release();
await this.workspaceCacheVersionService.incrementVersion(
await this.workspaceMetadataVersionService.getMetadataVersion(
fieldMetadataInput.workspaceId,
);
}
@ -426,7 +426,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
throw error;
} finally {
await queryRunner.release();
await this.workspaceCacheVersionService.incrementVersion(
await this.workspaceMetadataVersionService.incrementMetadataVersion(
fieldMetadataInput.workspaceId,
);
}
@ -545,7 +545,9 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
throw error;
} finally {
await queryRunner.release();
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
}
}
@ -586,7 +588,9 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
public async deleteFieldsMetadata(workspaceId: string) {
await this.fieldMetadataRepository.delete({ workspaceId });
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
}
private buildUpdatableStandardFieldInput(

6
packages/twenty-server/src/engine/metadata-modules/metadata-engine.module.ts
View File

@ -5,9 +5,9 @@ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { RelationMetadataModule } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.module';
import { RemoteServerModule } from 'src/engine/metadata-modules/remote-server/remote-server.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
@Module({
imports: [
@ -16,7 +16,7 @@ import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless
ObjectMetadataModule,
RelationMetadataModule,
ServerlessFunctionModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
WorkspaceMigrationModule,
RemoteServerModule,
],

4
packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts
View File

@ -19,7 +19,7 @@ import { ObjectMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadat
import { ObjectMetadataResolver } from 'src/engine/metadata-modules/object-metadata/object-metadata.resolver';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { RemoteTableRelationsModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.module';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
@ -43,7 +43,7 @@ import { UpdateObjectPayload } from './dtos/update-object.input';
DataSourceModule,
WorkspaceMigrationModule,
WorkspaceMigrationRunnerModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
FeatureFlagModule,
RemoteTableRelationsModule,
],

18
packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts
View File

@ -32,7 +32,7 @@ import {
import { RelationToDelete } from 'src/engine/metadata-modules/relation-metadata/types/relation-to-delete';
import { RemoteTableRelationsService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-relations/remote-table-relations.service';
import { mapUdtNameToFieldType } from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,
@ -80,7 +80,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
private readonly typeORMService: TypeORMService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
) {
super(objectMetadataRepository);
}
@ -144,7 +144,9 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
workspaceId,
);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return objectMetadata;
}
@ -352,7 +354,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
);
});
await this.workspaceCacheVersionService.incrementVersion(
await this.workspaceMetadataVersionService.incrementMetadataVersion(
objectMetadataInput.workspaceId,
);
@ -367,7 +369,9 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
const updatedObject = await super.updateOne(input.id, input.update);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return updatedObject;
}
@ -452,7 +456,9 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
public async deleteObjectsMetadata(workspaceId: string) {
await this.objectMetadataRepository.delete({ workspaceId });
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
}
private async createObjectRelationsMetadataAndMigrations(

4
packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.module.ts
View File

@ -12,7 +12,7 @@ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { RelationMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/relation-metadata/interceptors/relation-metadata-graphql-api-exception.interceptor';
import { RelationMetadataResolver } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.resolver';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
@ -34,7 +34,7 @@ import { RelationMetadataDTO } from './dtos/relation-metadata.dto';
FieldMetadataModule,
WorkspaceMigrationRunnerModule,
WorkspaceMigrationModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
],
services: [RelationMetadataService],
resolvers: [

10
packages/twenty-server/src/engine/metadata-modules/relation-metadata/relation-metadata.service.ts
View File

@ -22,7 +22,7 @@ import {
InvalidStringException,
validateMetadataNameOrThrow,
} from 'src/engine/metadata-modules/utils/validate-metadata-name.utils';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
WorkspaceMigrationColumnActionType,
@ -50,7 +50,7 @@ export class RelationMetadataService extends TypeOrmQueryService<RelationMetadat
private readonly fieldMetadataService: FieldMetadataService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
) {
super(relationMetadataRepository);
}
@ -120,7 +120,7 @@ export class RelationMetadataService extends TypeOrmQueryService<RelationMetadat
relationMetadataInput.workspaceId,
);
await this.workspaceCacheVersionService.incrementVersion(
await this.workspaceMetadataVersionService.incrementMetadataVersion(
relationMetadataInput.workspaceId,
);
@ -394,7 +394,9 @@ export class RelationMetadataService extends TypeOrmQueryService<RelationMetadat
relationMetadata.workspaceId,
);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
// TODO: Return id for delete endpoints
return relationMetadata;

4
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.module.ts
View File

@ -1,7 +1,7 @@
import { Module } from '@nestjs/common';
import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
@ -11,7 +11,7 @@ import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/wor
WorkspaceMigrationModule,
WorkspaceMigrationRunnerModule,
WorkspaceDataSourceModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
],
providers: [ForeignTableService],
exports: [ForeignTableService],

8
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service.ts
View File

@ -11,7 +11,7 @@ import {
} from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.exception';
import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/utils/get-foreign-table-column-name.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util';
import {
ReferencedTable,
@ -30,7 +30,7 @@ export class ForeignTableService {
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
) {}
public async fetchForeignTableNamesWithinWorkspace(
@ -127,7 +127,9 @@ export class ForeignTableService {
workspaceId,
);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return {
name: foreignTableName,

4
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts
View File

@ -11,7 +11,7 @@ import { RemoteTableSchemaUpdateModule } from 'src/engine/metadata-modules/remot
import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
@ -24,7 +24,7 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
DataSourceModule,
ObjectMetadataModule,
FieldMetadataModule,
WorkspaceCacheVersionModule,
WorkspaceMetadataVersionModule,
WorkspaceDataSourceModule,
ForeignTableModule,
RemoteTableSchemaUpdateModule,

66
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts
View File

@ -1,49 +1,49 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { plural } from 'pluralize';
import isEmpty from 'lodash.isempty';
import { plural } from 'pluralize';
import { Repository } from 'typeorm';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import {
RemoteServerType,
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service';
import { sortDistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import {
DistantTableUpdate,
RemoteTableStatus,
} from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import {
mapUdtNameToFieldSettings,
mapUdtNameToFieldType,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
import { camelCase } from 'src/utils/camel-case';
import { camelToTitleCase } from 'src/utils/camel-to-title-case';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util';
import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util';
import { ForeignTableService } from 'src/engine/metadata-modules/remote-server/remote-table/foreign-table/foreign-table.service';
import { RemoteTableSchemaUpdateService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table-schema-update/remote-table-schema-update.service';
import { sortDistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/utils/sort-distant-tables.util';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
import {
RemoteTableException,
RemoteTableExceptionCode,
} from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.exception';
import { fetchTableColumns } from 'src/engine/metadata-modules/remote-server/remote-table/utils/fetch-table-columns.util';
import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util';
import {
mapUdtNameToFieldSettings,
mapUdtNameToFieldType,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/udt-name-mapper.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import {
WorkspaceMigrationColumnAction,
WorkspaceMigrationColumnActionType,
} from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { camelCase } from 'src/utils/camel-case';
import { camelToTitleCase } from 'src/utils/camel-to-title-case';
export class RemoteTableService {
private readonly logger = new Logger(RemoteTableService.name);
@ -55,7 +55,7 @@ export class RemoteTableService {
private readonly remoteServerRepository: Repository<
RemoteServerEntity<RemoteServerType>
>,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService,
private readonly dataSourceService: DataSourceService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly fieldMetadataService: FieldMetadataService,
@ -250,7 +250,9 @@ export class RemoteTableService {
await this.remoteTableRepository.save(remoteTableEntity);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
return {
id: remoteTableEntity.id,
@ -435,7 +437,9 @@ export class RemoteTableService {
await this.remoteTableRepository.delete(remoteTable.id);
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
await this.workspaceMetadataVersionService.incrementMetadataVersion(
workspaceId,
);
}
private async createRemoteTableMetadata(

25
packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts
View File

@ -1,25 +0,0 @@
import {
Column,
CreateDateColumn,
Entity,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
@Entity('workspaceCacheVersion')
export class WorkspaceCacheVersionEntity {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ unique: true, nullable: false, type: 'uuid' })
workspaceId: string;
@Column()
version: string;
@CreateDateColumn({ type: 'timestamptz' })
createdAt: Date;
@UpdateDateColumn({ type: 'timestamptz' })
updatedAt: Date;
}

14
packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts
View File

@ -1,14 +0,0 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { WorkspaceCacheVersionEntity } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
@Module({
imports: [
TypeOrmModule.forFeature([WorkspaceCacheVersionEntity], 'metadata'),
],
exports: [WorkspaceCacheVersionService],
providers: [WorkspaceCacheVersionService],
})
export class WorkspaceCacheVersionModule {}

42
packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts
View File

@ -1,42 +0,0 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { WorkspaceCacheVersionEntity } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity';
@Injectable()
export class WorkspaceCacheVersionService {
constructor(
@InjectRepository(WorkspaceCacheVersionEntity, 'metadata')
private readonly workspaceCacheVersionRepository: Repository<WorkspaceCacheVersionEntity>,
) {}
async incrementVersion(workspaceId: string): Promise<string> {
const workspaceCacheVersion = (await this.getVersion(workspaceId)) ?? '0';
const newVersion = `${+workspaceCacheVersion + 1}`;
await this.workspaceCacheVersionRepository.upsert(
{
workspaceId,
version: `${+workspaceCacheVersion + 1}`,
},
['workspaceId'],
);
return newVersion;
}
async getVersion(workspaceId: string): Promise<string | null> {
const workspaceCacheVersion =
await this.workspaceCacheVersionRepository.findOne({
where: { workspaceId },
});
return workspaceCacheVersion?.version ?? null;
}
async deleteVersion(workspaceId: string): Promise<void> {
await this.workspaceCacheVersionRepository.delete({ workspaceId });
}
}

16
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts Normal file
View File

@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace], 'core'),
WorkspaceCacheStorageModule,
],
exports: [WorkspaceMetadataVersionService],
providers: [WorkspaceMetadataVersionService],
})
export class WorkspaceMetadataVersionModule {}

79
packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts Normal file
View File

@ -0,0 +1,79 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceMetadataVersionService {
logger = new Logger(WorkspaceMetadataVersionService.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
async flushCacheIfMetadataVersionIsOutdated(
workspaceId: string,
): Promise<void> {
const currentVersion =
(await this.workspaceCacheStorageService.getMetadataVersion(
workspaceId,
)) ?? 1;
let latestVersion = await this.getMetadataVersion(workspaceId);
if (latestVersion === undefined || currentVersion !== latestVersion) {
this.logger.log(
`Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`,
);
await this.workspaceCacheStorageService.flush(workspaceId);
latestVersion = await this.incrementMetadataVersion(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
latestVersion,
);
}
}
async incrementMetadataVersion(workspaceId: string): Promise<number> {
const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0;
const newMetadataVersion = metadataVersion + 1;
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: newMetadataVersion },
);
await this.workspaceCacheStorageService.setMetadataVersion(
workspaceId,
newMetadataVersion,
);
return newMetadataVersion;
}
async getMetadataVersion(workspaceId: string): Promise<number | undefined> {
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
});
return workspace?.metadataVersion;
}
async resetMetadataVersion(workspaceId: string): Promise<void> {
await this.workspaceRepository.update(
{ id: workspaceId },
{ metadataVersion: 1 },
);
await this.workspaceCacheStorageService.flush(workspaceId);
await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1);
}
}