Fix nested relations with large dataset in find queries (#7127)

## Before
<img width="920" alt="before"
src="https://github.com/user-attachments/assets/4809556f-0459-4f56-a716-b969a943d492">

## After
<img width="920" alt="after"
src="https://github.com/user-attachments/assets/504186b2-d002-482d-bc3e-2dda45c314b1">

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper.ts

@@ -0,0 +1,187 @@
+import {
+  FindManyOptions,
+  FindOptionsRelations,
+  In,
+  ObjectLiteral,
+} from 'typeorm';
+
+import { Record as IRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+
+import {
+  getRelationMetadata,
+  getRelationObjectMetadata,
+} from 'src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util';
+import {
+  ObjectMetadataMap,
+  ObjectMetadataMapItem,
+} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { deduceRelationDirection } from 'src/engine/utils/deduce-relation-direction.util';
+
+export class ProcessNestedRelationsHelper {
+  private readonly twentyORMGlobalManager: TwentyORMGlobalManager;
+
+  constructor(twentyORMGlobalManager: TwentyORMGlobalManager) {
+    this.twentyORMGlobalManager = twentyORMGlobalManager;
+  }
+
+  private async processFromRelation<ObjectRecord extends IRecord = IRecord>(
+    objectMetadataMap: ObjectMetadataMap,
+    parentObjectMetadataItem: ObjectMetadataMapItem,
+    parentObjectRecords: ObjectRecord[],
+    relationName: string,
+    nestedRelations: any,
+    limit: number,
+    authContext: any,
+  ) {
+    const relationFieldMetadata = parentObjectMetadataItem.fields[relationName];
+    const relationMetadata = getRelationMetadata(relationFieldMetadata);
+
+    const inverseRelationName =
+      objectMetadataMap[relationMetadata.toObjectMetadataId]?.fields[
+        relationMetadata.toFieldMetadataId
+      ]?.name;
+
+    const referenceObjectMetadata = getRelationObjectMetadata(
+      relationFieldMetadata,
+      objectMetadataMap,
+    );
+
+    const referenceObjectMetadataName = referenceObjectMetadata.nameSingular;
+
+    const relationRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        authContext.workspace.id,
+        referenceObjectMetadataName,
+      );
+
+    const relationIds = parentObjectRecords.map((item) => item.id);
+
+    const uniqueRelationIds = [...new Set(relationIds)];
+
+    const relationFindOptions: FindManyOptions = {
+      where: {
+        [`${inverseRelationName}Id`]: In(uniqueRelationIds),
+      },
+      take: limit * parentObjectRecords.length,
+    };
+
+    const relationResults = await relationRepository.find(relationFindOptions);
+
+    parentObjectRecords.forEach((item) => {
+      (item as any)[relationName] = relationResults.filter(
+        (rel) => rel[`${inverseRelationName}Id`] === item.id,
+      );
+    });
+
+    if (Object.keys(nestedRelations).length > 0) {
+      await this.processNestedRelations(
+        objectMetadataMap,
+        objectMetadataMap[relationName],
+        relationResults as ObjectRecord[],
+        nestedRelations as Record<string, FindOptionsRelations<ObjectLiteral>>,
+        limit,
+        authContext,
+      );
+    }
+  }
+
+  private async processToRelation<ObjectRecord extends IRecord = IRecord>(
+    objectMetadataMap: ObjectMetadataMap,
+    parentObjectMetadataItem: ObjectMetadataMapItem,
+    parentObjectRecords: ObjectRecord[],
+    relationName: string,
+    nestedRelations: any,
+    limit: number,
+    authContext: any,
+  ) {
+    const relationFieldMetadata = parentObjectMetadataItem.fields[relationName];
+
+    const referenceObjectMetadata = getRelationObjectMetadata(
+      relationFieldMetadata,
+      objectMetadataMap,
+    );
+
+    const referenceObjectMetadataName = referenceObjectMetadata.nameSingular;
+
+    const relationRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace(
+        authContext.workspace.id,
+        referenceObjectMetadataName,
+      );
+
+    const relationIds = parentObjectRecords.map(
+      (item) => item[`${relationName}Id`],
+    );
+
+    const uniqueRelationIds = [...new Set(relationIds)];
+
+    const relationFindOptions: FindManyOptions = {
+      where: {
+        id: In(uniqueRelationIds),
+      },
+      take: limit,
+    };
+
+    const relationResults = await relationRepository.find(relationFindOptions);
+
+    parentObjectRecords.forEach((item) => {
+      (item as any)[relationName] = relationResults.filter(
+        (rel) => rel.id === item[`${relationName}Id`],
+      )[0];
+    });
+
+    if (Object.keys(nestedRelations).length > 0) {
+      await this.processNestedRelations(
+        objectMetadataMap,
+        objectMetadataMap[relationName],
+        relationResults as ObjectRecord[],
+        nestedRelations as Record<string, FindOptionsRelations<ObjectLiteral>>,
+        limit,
+        authContext,
+      );
+    }
+  }
+
+  public async processNestedRelations<ObjectRecord extends IRecord = IRecord>(
+    objectMetadataMap: ObjectMetadataMap,
+    parentObjectMetadataItem: ObjectMetadataMapItem,
+    parentObjectRecords: ObjectRecord[],
+    relations: Record<string, FindOptionsRelations<ObjectLiteral>>,
+    limit: number,
+    authContext: any,
+  ) {
+    for (const [relationName, nestedRelations] of Object.entries(relations)) {
+      const relationFieldMetadata =
+        parentObjectMetadataItem.fields[relationName];
+      const relationMetadata = getRelationMetadata(relationFieldMetadata);
+
+      const relationDirection = deduceRelationDirection(
+        relationFieldMetadata,
+        relationMetadata,
+      );
+
+      if (relationDirection === 'to') {
+        await this.processToRelation(
+          objectMetadataMap,
+          parentObjectMetadataItem,
+          parentObjectRecords,
+          relationName,
+          nestedRelations,
+          limit,
+          authContext,
+        );
+      } else {
+        await this.processFromRelation(
+          objectMetadataMap,
+          parentObjectMetadataItem,
+          parentObjectRecords,
+          relationName,
+          nestedRelations,
+          limit,
+          authContext,
+        );
+      }
+    }
+  }
+}

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts

@@ -17,11 +17,15 @@ import {
   GraphqlQueryRunnerExceptionCode,
 } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
 import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser';
+import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper';
 import { ObjectRecordsToGraphqlConnectionMapper } from 'src/engine/api/graphql/graphql-query-runner/orm-mappers/object-records-to-graphql-connection.mapper';
 import { applyRangeFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/apply-range-filter.util';
 import { decodeCursor } from 'src/engine/api/graphql/graphql-query-runner/utils/cursors.util';
 import { getObjectMetadataOrThrow } from 'src/engine/api/graphql/graphql-query-runner/utils/get-object-metadata-or-throw.util';
-import { generateObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import {
+  generateObjectMetadataMap,
+  ObjectMetadataMapItem,
+} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 
 export class GraphqlQueryFindManyResolverService {
@@ -78,12 +82,12 @@ export class GraphqlQueryFindManyResolverService {
     const limit = args.first ?? args.last ?? QUERY_MAX_RECORDS;
 
     this.addOrderByColumnsToSelect(order, select);
+    this.addForeingKeyColumnsToSelect(relations, select, objectMetadata);
 
     const findOptions: FindManyOptions<ObjectLiteral> = {
       where,
       order,
       select,
-      relations,
       take: limit + 1,
     };
 
@@ -95,7 +99,10 @@ export class GraphqlQueryFindManyResolverService {
       applyRangeFilter(where, cursor, isForwardPagination);
     }
 
-    const objectRecords = await repository.find(findOptions);
+    const objectRecords = (await repository.find(
+      findOptions,
+    )) as ObjectRecord[];
+
     const { hasNextPage, hasPreviousPage } = this.getPaginationInfo(
       objectRecords,
       limit,
@@ -106,11 +113,26 @@ export class GraphqlQueryFindManyResolverService {
       objectRecords.pop();
     }
 
+    const processNestedRelationsHelper = new ProcessNestedRelationsHelper(
+      this.twentyORMGlobalManager,
+    );
+
+    if (relations) {
+      await processNestedRelationsHelper.processNestedRelations(
+        objectMetadataMap,
+        objectMetadata,
+        objectRecords,
+        relations,
+        limit,
+        authContext,
+      );
+    }
+
     const typeORMObjectRecordsParser =
       new ObjectRecordsToGraphqlConnectionMapper(objectMetadataMap);
 
     return typeORMObjectRecordsParser.createConnection(
-      objectRecords as ObjectRecord[],
+      objectRecords,
       objectMetadataItem.nameSingular,
       limit,
       totalCount,
@@ -179,6 +201,18 @@ export class GraphqlQueryFindManyResolverService {
     }
   }
 
+  private addForeingKeyColumnsToSelect(
+    relations: Record<string, any>,
+    select: Record<string, boolean>,
+    objectMetadata: ObjectMetadataMapItem,
+  ) {
+    for (const column of Object.keys(relations || {})) {
+      if (!select[`${column}Id`] && objectMetadata.fields[`${column}Id`]) {
+        select[`${column}Id`] = true;
+      }
+    }
+  }
+
   private getPaginationInfo(
     objectRecords: any[],
     limit: number,

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-one-resolver.service.ts

@@ -7,11 +7,13 @@ import {
 import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface';
 import { FindOneResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
 
+import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant';
 import {
   GraphqlQueryRunnerException,
   GraphqlQueryRunnerExceptionCode,
 } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
 import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser';
+import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper';
 import { ObjectRecordsToGraphqlConnectionMapper } from 'src/engine/api/graphql/graphql-query-runner/orm-mappers/object-records-to-graphql-connection.mapper';
 import { getObjectMetadataOrThrow } from 'src/engine/api/graphql/graphql-query-runner/utils/get-object-metadata-or-throw.util';
 import { generateObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
@@ -59,7 +61,11 @@ export class GraphqlQueryFindOneResolverService {
     );
     const where = graphqlQueryParser.parseFilter(args.filter ?? ({} as Filter));
 
-    const objectRecord = await repository.findOne({ where, select, relations });
+    const objectRecord = (await repository.findOne({
+      where,
+      select,
+    })) as ObjectRecord;
+    const limit = QUERY_MAX_RECORDS;
 
     if (!objectRecord) {
       throw new GraphqlQueryRunnerException(
@@ -68,11 +74,28 @@ export class GraphqlQueryFindOneResolverService {
       );
     }
 
+    const processNestedRelationsHelper = new ProcessNestedRelationsHelper(
+      this.twentyORMGlobalManager,
+    );
+
+    const objectRecords = [objectRecord];
+
+    if (relations) {
+      await processNestedRelationsHelper.processNestedRelations(
+        objectMetadataMap,
+        objectMetadata,
+        objectRecords,
+        relations,
+        limit,
+        authContext,
+      );
+    }
+
     const typeORMObjectRecordsParser =
       new ObjectRecordsToGraphqlConnectionMapper(objectMetadataMap);
 
     return typeORMObjectRecordsParser.processRecord(
-      objectRecord,
+      objectRecords[0],
       objectMetadataItem.nameSingular,
       1,
       1,

packages/twenty-server/src/engine/api/graphql/graphql-query-runner/utils/get-relation-object-metadata.util.ts

@@ -1,5 +1,6 @@
 import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
 
+import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
 import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
 import {
   deduceRelationDirection,
@@ -10,14 +11,7 @@ export const getRelationObjectMetadata = (
   fieldMetadata: FieldMetadataInterface,
   objectMetadataMap: ObjectMetadataMap,
 ) => {
-  const relationMetadata =
+  const relationMetadata = getRelationMetadata(fieldMetadata);
-    fieldMetadata.fromRelationMetadata ?? fieldMetadata.toRelationMetadata;
-
-  if (!relationMetadata) {
-    throw new Error(
-      `Relation metadata not found for field ${fieldMetadata.name}`,
-    );
-  }
 
   const relationDirection = deduceRelationDirection(
     fieldMetadata,
@@ -37,3 +31,18 @@ export const getRelationObjectMetadata = (
 
   return referencedObjectMetadata;
 };
+
+export const getRelationMetadata = (
+  fieldMetadata: FieldMetadataInterface,
+): RelationMetadataEntity => {
+  const relationMetadata =
+    fieldMetadata.fromRelationMetadata ?? fieldMetadata.toRelationMetadata;
+
+  if (!relationMetadata) {
+    throw new Error(
+      `Relation metadata not found for field ${fieldMetadata.name}`,
+    );
+  }
+
+  return relationMetadata;
+};