Expose duplicate check on REST API and enable batch duplicate checks (#6328)

This PR was created by [GitStart](https://gitstart.com/) to address the
requirements from this ticket:
[TWNTY-5472](https://clients.gitstart.com/twenty/5449/tickets/TWNTY-5472).
This ticket was imported from:
[TWNTY-5472](https://github.com/twentyhq/twenty/issues/5472)

 --- 

### Description:
- Following what is already done in the code, we create a REST endpoint
that generates the Graphql query and returns its result.

### Refs: #5472


### Demo:

<https://www.loom.com/share/e0b1030f056945a0bf93bdd88ea01d8f?sid=6f128e8c-370b-4079-958e-0ea2d073a241>

FIxes #5472

---------

Co-authored-by: gitstart-twenty <gitstart-twenty@users.noreply.github.com>
Co-authored-by: gitstart-twenty <140154534+gitstart-twenty@users.noreply.github.com>
Co-authored-by: martmull <martmull@hotmail.fr>

packages/twenty-server/src/engine/api/rest/core/controllers/rest-api-core.controller.ts

@@ -18,6 +18,13 @@ import { cleanGraphQLResponse } from 'src/engine/api/rest/utils/clean-graphql-re
 export class RestApiCoreController {
   constructor(private readonly restApiCoreService: RestApiCoreService) {}
 
+  @Post('/duplicates')
+  async handleApiFindDuplicates(@Req() request: Request, @Res() res: Response) {
+    const result = await this.restApiCoreService.findDuplicates(request);
+
+    res.status(200).send(cleanGraphQLResponse(result.data.data));
+  }
+
   @Get()
   async handleApiGet(@Req() request: Request, @Res() res: Response) {
     const result = await this.restApiCoreService.get(request);

packages/twenty-server/src/engine/api/rest/core/query-builder/core-query-builder.factory.ts

@@ -2,24 +2,26 @@ import { BadRequestException, Injectable } from '@nestjs/common';
 
 import { Request } from 'express';
 
-import { DeleteQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-query.factory';
-import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
-import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
-import { CreateOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/create-one-query.factory';
-import { UpdateQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/update-query.factory';
-import { FindOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-one-query.factory';
-import { FindManyQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-many-query.factory';
-import { DeleteVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-variables.factory';
-import { CreateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/create-variables.factory';
-import { UpdateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/update-variables.factory';
-import { GetVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/get-variables.factory';
-import { parseCorePath } from 'src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils';
-import { computeDepth } from 'src/engine/api/rest/core/query-builder/utils/compute-depth.utils';
-import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
-import { Query } from 'src/engine/api/rest/core/types/query.type';
-import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
 import { CreateManyQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/create-many-query.factory';
+import { CreateOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/create-one-query.factory';
+import { CreateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/create-variables.factory';
+import { DeleteQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-query.factory';
+import { DeleteVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-variables.factory';
+import { FindDuplicatesQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-duplicates-query.factory';
+import { FindDuplicatesVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/find-duplicates-variables.factory';
+import { FindManyQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-many-query.factory';
+import { FindOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-one-query.factory';
+import { GetVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/get-variables.factory';
+import { UpdateQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/update-query.factory';
+import { UpdateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/update-variables.factory';
+import { computeDepth } from 'src/engine/api/rest/core/query-builder/utils/compute-depth.utils';
 import { parseCoreBatchPath } from 'src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-batch-path.utils';
+import { parseCorePath } from 'src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils';
+import { Query } from 'src/engine/api/rest/core/types/query.type';
+import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
+import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
 
 @Injectable()
 export class CoreQueryBuilderFactory {
@@ -30,10 +32,12 @@ export class CoreQueryBuilderFactory {
     private readonly updateQueryFactory: UpdateQueryFactory,
     private readonly findOneQueryFactory: FindOneQueryFactory,
     private readonly findManyQueryFactory: FindManyQueryFactory,
+    private readonly findDuplicatesQueryFactory: FindDuplicatesQueryFactory,
     private readonly deleteVariablesFactory: DeleteVariablesFactory,
     private readonly createVariablesFactory: CreateVariablesFactory,
     private readonly updateVariablesFactory: UpdateVariablesFactory,
     private readonly getVariablesFactory: GetVariablesFactory,
+    private readonly findDuplicatesVariablesFactory: FindDuplicatesVariablesFactory,
     private readonly objectMetadataService: ObjectMetadataService,
     private readonly tokenService: TokenService,
     private readonly environmentService: EnvironmentService,
@@ -161,4 +165,15 @@ export class CoreQueryBuilderFactory {
       variables: this.getVariablesFactory.create(id, request, objectMetadata),
     };
   }
+
+  async findDuplicates(request: Request): Promise<Query> {
+    const { object: parsedObject } = parseCorePath(request);
+    const objectMetadata = await this.getObjectMetadata(request, parsedObject);
+    const depth = computeDepth(request);
+
+    return {
+      query: this.findDuplicatesQueryFactory.create(objectMetadata, depth),
+      variables: this.findDuplicatesVariablesFactory.create(request),
+    };
+  }
 }

packages/twenty-server/src/engine/api/rest/core/query-builder/factories/factories.ts

@@ -1,13 +1,15 @@
-import { DeleteQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-query.factory';
-import { CreateOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/create-one-query.factory';
-import { UpdateQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/update-query.factory';
-import { FindOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-one-query.factory';
-import { FindManyQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-many-query.factory';
-import { DeleteVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-variables.factory';
-import { CreateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/create-variables.factory';
-import { UpdateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/update-variables.factory';
-import { GetVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/get-variables.factory';
 import { CreateManyQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/create-many-query.factory';
+import { CreateOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/create-one-query.factory';
+import { CreateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/create-variables.factory';
+import { DeleteQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-query.factory';
+import { DeleteVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/delete-variables.factory';
+import { FindDuplicatesQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-duplicates-query.factory';
+import { FindDuplicatesVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/find-duplicates-variables.factory';
+import { FindManyQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-many-query.factory';
+import { FindOneQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/find-one-query.factory';
+import { GetVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/get-variables.factory';
+import { UpdateQueryFactory } from 'src/engine/api/rest/core/query-builder/factories/update-query.factory';
+import { UpdateVariablesFactory } from 'src/engine/api/rest/core/query-builder/factories/update-variables.factory';
 import { inputFactories } from 'src/engine/api/rest/input-factories/factories';
 
 export const coreQueryBuilderFactories = [
@@ -17,9 +19,11 @@ export const coreQueryBuilderFactories = [
   UpdateQueryFactory,
   FindOneQueryFactory,
   FindManyQueryFactory,
+  FindDuplicatesQueryFactory,
   DeleteVariablesFactory,
   CreateVariablesFactory,
   UpdateVariablesFactory,
   GetVariablesFactory,
+  FindDuplicatesVariablesFactory,
   ...inputFactories,
 ];

packages/twenty-server/src/engine/api/rest/core/query-builder/factories/find-duplicates-query.factory.ts

@@ -0,0 +1,39 @@
+import { Injectable } from '@nestjs/common';
+
+import { mapFieldMetadataToGraphqlQuery } from 'src/engine/api/rest/core/query-builder/utils/map-field-metadata-to-graphql-query.utils';
+import { capitalize } from 'src/utils/capitalize';
+
+@Injectable()
+export class FindDuplicatesQueryFactory {
+  create(objectMetadata, depth?: number): string {
+    const objectNameSingular = objectMetadata.objectMetadataItem.nameSingular;
+
+    return `
+      query FindDuplicate${capitalize(
+        objectNameSingular,
+      )}($ids: [ID], $data: [${capitalize(objectNameSingular)}CreateInput]) {
+        ${objectNameSingular}Duplicates(ids: $ids, data: $data) {
+          totalCount
+          pageInfo {
+            hasNextPage
+            startCursor
+            endCursor
+          }
+          edges{
+            node {
+                ${objectMetadata.objectMetadataItem.fields
+                  .map((field) =>
+                    mapFieldMetadataToGraphqlQuery(
+                      objectMetadata.objectMetadataItems,
+                      field,
+                      depth,
+                    ),
+                  )
+                  .join('\n')}
+            }
+          }
+        }
+      }
+    `;
+  }
+}

packages/twenty-server/src/engine/api/rest/core/query-builder/factories/find-duplicates-variables.factory.ts

@@ -0,0 +1,12 @@
+import { Injectable } from '@nestjs/common';
+
+import { Request } from 'express';
+
+import { QueryVariables } from 'src/engine/api/rest/core/types/query-variables.type';
+
+@Injectable()
+export class FindDuplicatesVariablesFactory {
+  create(request: Request): QueryVariables {
+    return request.body;
+  }
+}

packages/twenty-server/src/engine/api/rest/core/rest-api-core.service.ts

@@ -44,4 +44,10 @@ export class RestApiCoreService {
 
     return await this.restApiService.call(GraphqlApiType.CORE, request, data);
   }
+
+  async findDuplicates(request: Request) {
+    const data = await this.coreQueryBuilderFactory.findDuplicates(request);
+
+    return await this.restApiService.call(GraphqlApiType.CORE, request, data);
+  }
 }

packages/twenty-server/src/engine/api/rest/core/types/query-variables.type.ts

@@ -1,5 +1,6 @@
 export type QueryVariables = {
   id?: string;
+  ids?: string[];
   data?: object | null;
   filter?: object;
   orderBy?: object;

packages/twenty-server/src/engine/api/rest/utils/__tests__/clean-graphql-response.utils.spec.ts

@@ -72,4 +72,94 @@ describe('cleanGraphQLResponse', () => {
 
     expect(cleanGraphQLResponse(data)).toEqual(expectedResult);
   });
+
+  it('should remove nested edges/node from results if data key is an array', () => {
+    const data = {
+      companyDuplicates: [
+        {
+          totalCount: 14,
+          pageInfo: {
+            hasNextPage: true,
+            startCursor:
+              'WyIwMDliYjNkYy1hNGEyLTRiNWUtYTZmYi1iMTFiMmFlMGI1MmIiXQ==',
+            endCursor:
+              'WyIyMDIwMjAyMC0wNzEzLTQwYTUtODIxNi04MjgwMjQwMWQzM2UiXQ==',
+          },
+          edges: [
+            {
+              node: {
+                id: 'id',
+                createdAt: '2023-01-01',
+                people: {
+                  edges: [{ node: { id: 'id1' } }, { node: { id: 'id2' } }],
+                },
+              },
+            },
+          ],
+        },
+        {
+          totalCount: 14,
+          pageInfo: {
+            hasNextPage: true,
+            startCursor:
+              'WyIwMDliYjNkYy1hNGEyLTRiNWUtYTZmYi1iMTFiMmFlMGI1MmIiXQ==',
+            endCursor:
+              'WyIyMDIwMjAyMC0wNzEzLTQwYTUtODIxNi04MjgwMjQwMWQzM2UiXQ==',
+          },
+          edges: [
+            {
+              node: {
+                id: 'id',
+                createdAt: '2023-01-01',
+                people: {
+                  edges: [{ node: { id: 'id1' } }, { node: { id: 'id2' } }],
+                },
+              },
+            },
+          ],
+        },
+      ],
+    };
+
+    const expectedResult = {
+      data: [
+        {
+          totalCount: 14,
+          pageInfo: {
+            hasNextPage: true,
+            startCursor:
+              'WyIwMDliYjNkYy1hNGEyLTRiNWUtYTZmYi1iMTFiMmFlMGI1MmIiXQ==',
+            endCursor:
+              'WyIyMDIwMjAyMC0wNzEzLTQwYTUtODIxNi04MjgwMjQwMWQzM2UiXQ==',
+          },
+          companyDuplicates: [
+            {
+              id: 'id',
+              createdAt: '2023-01-01',
+              people: [{ id: 'id1' }, { id: 'id2' }],
+            },
+          ],
+        },
+        {
+          totalCount: 14,
+          pageInfo: {
+            hasNextPage: true,
+            startCursor:
+              'WyIwMDliYjNkYy1hNGEyLTRiNWUtYTZmYi1iMTFiMmFlMGI1MmIiXQ==',
+            endCursor:
+              'WyIyMDIwMjAyMC0wNzEzLTQwYTUtODIxNi04MjgwMjQwMWQzM2UiXQ==',
+          },
+          companyDuplicates: [
+            {
+              id: 'id',
+              createdAt: '2023-01-01',
+              people: [{ id: 'id1' }, { id: 'id2' }],
+            },
+          ],
+        },
+      ],
+    };
+
+    expect(cleanGraphQLResponse(data)).toEqual(expectedResult);
+  });
 });

packages/twenty-server/src/engine/api/rest/utils/clean-graphql-response.utils.ts

@@ -43,6 +43,16 @@ export const cleanGraphQLResponse = (input: any) => {
     } else if (isObject(input[key])) {
       // Recursively clean and assign nested objects under the data key
       output.data[key] = cleanObject(input[key]);
+    } else if (Array.isArray(input[key])) {
+      const itemsWithEdges = input[key].filter((item) => item.edges);
+      const cleanedObjArray = itemsWithEdges.map(({ edges, ...item }) => {
+        return {
+          ...item,
+          [key]: edges.map((edge) => cleanObject(edge.node)),
+        };
+      });
+
+      output.data = cleanedObjArray;
     } else {
       // Assign all other properties directly under the data key
       output.data[key] = input[key];

packages/twenty-server/src/engine/core-modules/open-api/open-api.service.ts

@@ -4,17 +4,7 @@ import { Request } from 'express';
 import { OpenAPIV3_1 } from 'openapi-types';
 
 import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
-import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
 import { baseSchema } from 'src/engine/core-modules/open-api/utils/base-schema.utils';
-import {
-  computeBatchPath,
-  computeManyResultPath,
-  computeSingleResultPath,
-} from 'src/engine/core-modules/open-api/utils/path.utils';
-import {
-  get400ErrorResponses,
-  get401ErrorResponses,
-} from 'src/engine/core-modules/open-api/utils/get-error-responses.utils';
 import {
   computeMetadataSchemaComponents,
   computeParameterComponents,
@@ -22,16 +12,27 @@ import {
 } from 'src/engine/core-modules/open-api/utils/components.utils';
 import { computeSchemaTags } from 'src/engine/core-modules/open-api/utils/compute-schema-tags.utils';
 import { computeWebhooks } from 'src/engine/core-modules/open-api/utils/computeWebhooks.utils';
-import { capitalize } from 'src/utils/capitalize';
 import {
+  get400ErrorResponses,
+  get401ErrorResponses,
+} from 'src/engine/core-modules/open-api/utils/get-error-responses.utils';
+import {
+  computeBatchPath,
+  computeDuplicatesResultPath,
+  computeManyResultPath,
+  computeSingleResultPath,
+} from 'src/engine/core-modules/open-api/utils/path.utils';
+import { getRequestBody } from 'src/engine/core-modules/open-api/utils/request-body.utils';
+import {
+  getCreateOneResponse201,
   getDeleteResponse200,
   getFindManyResponse200,
-  getCreateOneResponse201,
   getFindOneResponse200,
   getUpdateOneResponse200,
 } from 'src/engine/core-modules/open-api/utils/responses.utils';
-import { getRequestBody } from 'src/engine/core-modules/open-api/utils/request-body.utils';
 import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
+import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
+import { capitalize } from 'src/utils/capitalize';
 import { getServerUrl } from 'src/utils/get-server-url';
 
 @Injectable()
@@ -68,6 +69,8 @@ export class OpenApiService {
       paths[`/${item.namePlural}`] = computeManyResultPath(item);
       paths[`/batch/${item.namePlural}`] = computeBatchPath(item);
       paths[`/${item.namePlural}/{id}`] = computeSingleResultPath(item);
+      paths[`/${item.namePlural}/duplicates`] =
+        computeDuplicatesResultPath(item);
 
       return paths;
     }, schema.paths as OpenAPIV3_1.PathsObject);

packages/twenty-server/src/engine/core-modules/open-api/utils/path.utils.ts

@@ -1,20 +1,22 @@
 import { OpenAPIV3_1 } from 'openapi-types';
 
-import { capitalize } from 'src/utils/capitalize';
-import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
-import {
-  getDeleteResponse200,
-  getJsonResponse,
-  getFindManyResponse200,
-  getCreateOneResponse201,
-  getCreateManyResponse201,
-  getFindOneResponse200,
-  getUpdateOneResponse200,
-} from 'src/engine/core-modules/open-api/utils/responses.utils';
 import {
   getArrayRequestBody,
+  getFindDuplicatesRequestBody,
   getRequestBody,
 } from 'src/engine/core-modules/open-api/utils/request-body.utils';
+import {
+  getCreateManyResponse201,
+  getCreateOneResponse201,
+  getDeleteResponse200,
+  getFindDuplicatesResponse200,
+  getFindManyResponse200,
+  getFindOneResponse200,
+  getJsonResponse,
+  getUpdateOneResponse200,
+} from 'src/engine/core-modules/open-api/utils/responses.utils';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { capitalize } from 'src/utils/capitalize';
 
 export const computeBatchPath = (
   item: ObjectMetadataEntity,
@@ -140,3 +142,23 @@ export const computeOpenApiPath = (
     },
   } as OpenAPIV3_1.PathItemObject;
 };
+
+export const computeDuplicatesResultPath = (
+  item: ObjectMetadataEntity,
+): OpenAPIV3_1.PathItemObject => {
+  return {
+    post: {
+      tags: [item.namePlural],
+      summary: `Find ${item.nameSingular} Duplicates`,
+      description: `**depth** can be provided to request your **${item.nameSingular}**`,
+      operationId: `find${capitalize(item.nameSingular)}Duplicates`,
+      parameters: [{ $ref: '#/components/parameters/depth' }],
+      requestBody: getFindDuplicatesRequestBody(capitalize(item.nameSingular)),
+      responses: {
+        '200': getFindDuplicatesResponse200(item),
+        '400': { $ref: '#/components/responses/400' },
+        '401': { $ref: '#/components/responses/401' },
+      },
+    },
+  } as OpenAPIV3_1.PathItemObject;
+};

packages/twenty-server/src/engine/core-modules/open-api/utils/request-body.utils.ts

@@ -27,3 +27,32 @@ export const getArrayRequestBody = (name: string) => {
     },
   };
 };
+
+export const getFindDuplicatesRequestBody = (name: string) => {
+  return {
+    description: 'body',
+    required: true,
+    content: {
+      'application/json': {
+        schema: {
+          type: 'object',
+          properties: {
+            data: {
+              type: 'array',
+              items: {
+                $ref: `#/components/schemas/${name}`,
+              },
+            },
+            ids: {
+              type: 'array',
+              items: {
+                type: 'string',
+                format: 'uuid',
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+};

packages/twenty-server/src/engine/core-modules/open-api/utils/responses.utils.ts

@@ -298,3 +298,45 @@ export const getJsonResponse = () => {
     },
   };
 };
+
+export const getFindDuplicatesResponse200 = (
+  item: Pick<ObjectMetadataEntity, 'nameSingular'>,
+) => {
+  return {
+    description: 'Successful operation',
+    content: {
+      'application/json': {
+        schema: {
+          type: 'object',
+          properties: {
+            data: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  totalCount: { type: 'number' },
+                  pageInfo: {
+                    type: 'object',
+                    properties: {
+                      hasNextPage: { type: 'boolean' },
+                      startCursor: { type: 'string' },
+                      endCursor: { type: 'string' },
+                    },
+                  },
+                  companyDuplicates: {
+                    type: 'array',
+                    items: {
+                      $ref: `#/components/schemas/${capitalize(
+                        item.nameSingular,
+                      )}`,
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+};