Optimize metadata queries (#7013)

In this PR: 1. Refactor guards to avoid duplicated queries: WorkspaceAuthGuard and UserAuthGuard only check for existence of workspace and user in the request without querying the database
2024-09-13 19:11:32 +02:00
parent cf8b1161cc
commit 523df5398a
132 changed files with 818 additions and 6372 deletions
--- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto.ts
@ -44,7 +44,7 @@ registerEnumType(FieldMetadataType, {
@ObjectType('field')
@Authorize({
  authorize: (context: any) => ({
-    workspaceId: { eq: context?.req?.user?.workspace?.id },
+    workspaceId: { eq: context?.req?.workspace?.id },
  }),
 })
@QueryOptions({
@ -132,6 +132,8 @@ export class FieldMetadataDTO<
  @HideField()
  workspaceId: string;

+  objectMetadataId: string;
+
  @IsDateString()
  @Field()
  createdAt: Date;
--- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts
@ -9,13 +9,14 @@ import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm';

 import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { ActorModule } from 'src/engine/core-modules/actor/actor.module';
-import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
+import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
 import { FieldMetadataResolver } from 'src/engine/metadata-modules/field-metadata/field-metadata.resolver';
 import { FieldMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/field-metadata/interceptors/field-metadata-graphql-api-exception.interceptor';
 import { IsFieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-default-value.validator';
 import { IsFieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-options.validator';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
 import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module';
 import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module';
@ -32,7 +33,10 @@ import { UpdateFieldInput } from './dtos/update-field.input';
  imports: [
    NestjsQueryGraphQLModule.forFeature({
      imports: [
-        NestjsQueryTypeOrmModule.forFeature([FieldMetadataEntity], 'metadata'),
+        NestjsQueryTypeOrmModule.forFeature(
+          [FieldMetadataEntity, ObjectMetadataEntity],
+          'metadata',
+        ),
        WorkspaceMigrationModule,
        WorkspaceStatusModule,
        WorkspaceMigrationRunnerModule,
@ -65,7 +69,7 @@ import { UpdateFieldInput } from './dtos/update-field.input';
            many: { disabled: true },
          },
          delete: { disabled: true },
-          guards: [JwtAuthGuard],
+          guards: [WorkspaceAuthGuard],
          interceptors: [FieldMetadataGraphqlApiExceptionInterceptor],
        },
      ],
--- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.resolver.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.resolver.ts
@ -15,7 +15,7 @@ import {
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
 import { IDataloaders } from 'src/engine/dataloaders/dataloader.interface';
 import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
-import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
+import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
 import { CreateOneFieldMetadataInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
 import { DeleteOneFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/delete-field.input';
 import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
@ -25,7 +25,7 @@ import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/fi
 import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
 import { fieldMetadataGraphqlApiExceptionHandler } from 'src/engine/metadata-modules/field-metadata/utils/field-metadata-graphql-api-exception-handler.util';

-@UseGuards(JwtAuthGuard)
+@UseGuards(WorkspaceAuthGuard)
@Resolver(() => FieldMetadataDTO)
 export class FieldMetadataResolver {
  constructor(private readonly fieldMetadataService: FieldMetadataService) {}
@ -103,6 +103,7 @@ export class FieldMetadataResolver {

  @ResolveField(() => RelationDefinitionDTO, { nullable: true })
  async relationDefinition(
+    @AuthWorkspace() workspace: Workspace,
    @Parent() fieldMetadata: FieldMetadataDTO,
    @Context() context: { loaders: IDataloaders },
  ): Promise<RelationDefinitionDTO | null | undefined> {
@ -112,7 +113,10 @@ export class FieldMetadataResolver {

    try {
      const relationMetadataItem =
-        await context.loaders.relationMetadataLoader.load(fieldMetadata.id);
+        await context.loaders.relationMetadataLoader.load({
+          fieldMetadata,
+          workspaceId: workspace.id,
+        });

      return await this.fieldMetadataService.getRelationDefinitionFromRelationMetadata(
        fieldMetadata,
--- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
@ -4,7 +4,7 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
 import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
 import isEmpty from 'lodash.isempty';
 import { DataSource, FindOneOptions, Repository } from 'typeorm';
-import { v4 as uuidV4 } from 'uuid';
+import { v4 as uuidV4, v4 } from 'uuid';

 import { TypeORMService } from 'src/database/typeorm/typeorm.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@ -72,6 +72,8 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
    private readonly metadataDataSource: DataSource,
    @InjectRepository(FieldMetadataEntity, 'metadata')
    private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
+    @InjectRepository(ObjectMetadataEntity, 'metadata')
+    private readonly objectMetadataRepository: Repository<ObjectMetadataEntity>,
    private readonly objectMetadataService: ObjectMetadataService,
    private readonly workspaceMigrationFactory: WorkspaceMigrationFactory,
    private readonly workspaceMigrationService: WorkspaceMigrationService,
@ -87,6 +89,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
  override async createOne(
    fieldMetadataInput: CreateFieldInput,
  ): Promise<FieldMetadataEntity> {
+    console.time('createOne');
    const queryRunner = this.metadataDataSource.createQueryRunner();

    await queryRunner.connect();
@ -97,20 +100,23 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
        queryRunner.manager.getRepository<FieldMetadataEntity>(
          FieldMetadataEntity,
        );
-      const objectMetadata =
-        await this.objectMetadataService.findOneWithinWorkspace(
-          fieldMetadataInput.workspaceId,
-          {
-            where: {
-              id: fieldMetadataInput.objectMetadataId,
-            },
-          },
-        );
+
+      console.time('createOne query');
+      const [objectMetadata] = await this.objectMetadataRepository.find({
+        where: {
+          id: fieldMetadataInput.objectMetadataId,
+          workspaceId: fieldMetadataInput.workspaceId,
+        },
+        relations: ['fields'],
+        order: {},
+      });
+
+      console.timeEnd('createOne query');

      if (!objectMetadata) {
        throw new FieldMetadataException(
          'Object metadata does not exist',
-          FieldMetadataExceptionCode.INVALID_FIELD_INPUT,
+          FieldMetadataExceptionCode.OBJECT_METADATA_NOT_FOUND,
        );
      }

@ -155,22 +161,11 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
        objectMetadata,
      );

-      const fieldAlreadyExists = await fieldMetadataRepository.findOne({
-        where: {
-          name: fieldMetadataInput.name,
-          objectMetadataId: fieldMetadataInput.objectMetadataId,
-          workspaceId: fieldMetadataInput.workspaceId,
-        },
-      });
-
-      if (fieldAlreadyExists) {
-        throw new FieldMetadataException(
-          'Field already exists',
-          FieldMetadataExceptionCode.FIELD_ALREADY_EXISTS,
-        );
-      }
-
+      console.time('createOne save');
      const createdFieldMetadata = await fieldMetadataRepository.save({
+        id: v4(),
+        createdAt: new Date(),
+        updatedAt: new Date(),
        ...fieldMetadataInput,
        isNullable: generateNullable(
          fieldMetadataInput.type,
@ -190,7 +185,10 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
        isCustom: true,
      });

+      console.timeEnd('createOne save');
+
      if (!fieldMetadataInput.isRemoteCreation) {
+        console.time('createOne migration create');
        await this.workspaceMigrationService.createCustomMigration(
          generateMigrationName(`create-${createdFieldMetadata.name}`),
          fieldMetadataInput.workspaceId,
@ -206,11 +204,16 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
          ],
        );

+        console.timeEnd('createOne migration create');
+
+        console.time('createOne migration run');
        await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
          fieldMetadataInput.workspaceId,
        );
+        console.timeEnd('createOne migration run');
      }

+      console.time('createOne workspace viewField');
      // TODO: Move viewField creation to a cdc scheduler
      const dataSourceMetadata =
        await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
@ -270,8 +273,11 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
            );
          }
        }
+        console.timeEnd('createOne workspace viewField');

+        console.time('createOne internal commit');
        await workspaceQueryRunner.commitTransaction();
+        console.timeEnd('createOne internal commit');
      } catch (error) {
        await workspaceQueryRunner.rollbackTransaction();
        throw error;
@ -279,7 +285,9 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
        await workspaceQueryRunner.release();
      }

+      console.time('createOne commit');
      await queryRunner.commitTransaction();
+      console.timeEnd('createOne commit');

      return createdFieldMetadata;
    } catch (error) {
@ -287,9 +295,12 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
      throw error;
    } finally {
      await queryRunner.release();
+      console.time('createOne increment');
      await this.workspaceMetadataVersionService.incrementMetadataVersion(
        fieldMetadataInput.workspaceId,
      );
+      console.timeEnd('createOne increment');
+      console.timeEnd('createOne');
    }
  }

@ -308,7 +319,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
          FieldMetadataEntity,
        );

-      const existingFieldMetadata = await fieldMetadataRepository.findOne({
+      const [existingFieldMetadata] = await fieldMetadataRepository.find({
        where: {
          id,
          workspaceId: fieldMetadataInput.workspaceId,
@ -322,15 +333,14 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
        );
      }

-      const objectMetadata =
-        await this.objectMetadataService.findOneWithinWorkspace(
-          fieldMetadataInput.workspaceId,
-          {
-            where: {
-              id: existingFieldMetadata?.objectMetadataId,
-            },
-          },
-        );
+      const [objectMetadata] = await this.objectMetadataRepository.find({
+        where: {
+          id: existingFieldMetadata.objectMetadataId,
+          workspaceId: fieldMetadataInput.workspaceId,
+        },
+        relations: ['fields'],
+        order: {},
+      });

      if (!objectMetadata) {
        throw new FieldMetadataException(
@ -475,7 +485,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
          FieldMetadataEntity,
        );

-      const fieldMetadata = await fieldMetadataRepository.findOne({
+      const [fieldMetadata] = await fieldMetadataRepository.find({
        where: {
          id: input.id,
          workspaceId: workspaceId,
@ -489,12 +499,13 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
        );
      }

-      const objectMetadata =
-        await this.objectMetadataService.findOneWithinWorkspace(workspaceId, {
-          where: {
-            id: fieldMetadata?.objectMetadataId,
-          },
-        });
+      const [objectMetadata] = await this.objectMetadataRepository.find({
+        where: {
+          id: fieldMetadata.objectMetadataId,
+        },
+        relations: ['fields'],
+        order: {},
+      });

      if (!objectMetadata) {
        throw new FieldMetadataException(
@ -583,7 +594,7 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
    id: string,
    options?: FindOneOptions<FieldMetadataEntity>,
  ) {
-    const fieldMetadata = await this.fieldMetadataRepository.findOne({
+    const [fieldMetadata] = await this.fieldMetadataRepository.find({
      ...options,
      where: {
        ...options?.where,
@ -605,13 +616,15 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
    workspaceId: string,
    options: FindOneOptions<FieldMetadataEntity>,
  ) {
-    return this.fieldMetadataRepository.findOne({
+    const [fieldMetadata] = await this.fieldMetadataRepository.find({
      ...options,
      where: {
        ...options.where,
        workspaceId,
      },
    });
+
+    return fieldMetadata;
  }

  private buildUpdatableStandardFieldInput(