Upsert endpoint and CSV import upsert (#5970)

This PR introduces an `upsert` parameter (along the existing `data`
param) for `createOne` and `createMany` mutations.

When upsert is set to `true`, the function will look for records with
the same id if an id was passed. If not id was passed, it will leverage
the existing duplicate check mechanism to find a duplicate. If a record
is found, then the function will perform an update instead of a create.

Unfortunately I had to remove some nice tests that existing on the args
factory. Those tests where mostly testing the duplication rule
generation logic but through a GraphQL angle. Since I moved the
duplication rule logic to a dedicated service, if I kept the tests but
mocked the service we wouldn't really be testing anything useful. The
right path would be to create new tests for this service that compare
the JSON output and not the GraphQL output but I chose not to work on
this as it's equivalent to rewriting the tests from scratch and I have
other competing priorities.

packages/twenty-server/src/engine/core-modules/duplicate/constants/duplicate-criteria.constants.ts

@@ -0,0 +1,30 @@
+import { RecordDuplicateCriteria } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+
+/**
+ * objectName: directly reference the name of the object from the metadata tables.
+ * columnNames: reference the column names not the field names.
+ * So if we need to reference a custom field, we should directly add the column name like `_customColumn`.
+ * If we need to terence a composite field, we should add all children of the composite like `nameFirstName` and `nameLastName`
+ */
+export const DUPLICATE_CRITERIA_COLLECTION: RecordDuplicateCriteria[] = [
+  {
+    objectName: 'company',
+    columnNames: ['domainName'],
+  },
+  {
+    objectName: 'company',
+    columnNames: ['name'],
+  },
+  {
+    objectName: 'person',
+    columnNames: ['nameFirstName', 'nameLastName'],
+  },
+  {
+    objectName: 'person',
+    columnNames: ['linkedinLinkUrl'],
+  },
+  {
+    objectName: 'person',
+    columnNames: ['email'],
+  },
+];

packages/twenty-server/src/engine/core-modules/duplicate/duplicate.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+
+import { DuplicateService } from 'src/engine/core-modules/duplicate/duplicate.service';
+import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
+
+@Module({
+  imports: [WorkspaceDataSourceModule],
+  exports: [DuplicateService],
+  providers: [DuplicateService],
+})
+export class DuplicateModule {}

packages/twenty-server/src/engine/core-modules/duplicate/duplicate.service.ts

@@ -0,0 +1,173 @@
+import { Injectable } from '@nestjs/common';
+
+import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
+import {
+  Record as IRecord,
+  Record,
+} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+
+import { settings } from 'src/engine/constants/settings';
+import { computeObjectTargetTable } from 'src/engine/utils/compute-object-target-table.util';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
+import { DUPLICATE_CRITERIA_COLLECTION } from 'src/engine/core-modules/duplicate/constants/duplicate-criteria.constants';
+
+@Injectable()
+export class DuplicateService {
+  constructor(
+    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
+  ) {}
+
+  async findExistingRecords(
+    recordIds: (string | number)[],
+    objectMetadata: ObjectMetadataInterface,
+    workspaceId: string,
+  ) {
+    const dataSourceSchema =
+      this.workspaceDataSourceService.getSchemaName(workspaceId);
+
+    const results = await this.workspaceDataSourceService.executeRawQuery(
+      `
+          SELECT 
+             *
+          FROM
+              ${dataSourceSchema}."${computeObjectTargetTable(
+                objectMetadata,
+              )}" p
+          WHERE
+              p."id" IN (${recordIds
+                .map((_, index) => `$${index + 1}`)
+                .join(', ')})
+          `,
+      recordIds,
+      workspaceId,
+    );
+
+    return results as IRecord[];
+  }
+
+  buildDuplicateConditionForGraphQL(
+    objectMetadata: ObjectMetadataInterface,
+    argsData?: Partial<Record>,
+    filteringByExistingRecordId?: string,
+  ) {
+    if (!argsData) {
+      return;
+    }
+
+    const criteriaCollection =
+      this.getApplicableDuplicateCriteriaCollection(objectMetadata);
+
+    const criteriaWithMatchingArgs = criteriaCollection.filter((criteria) =>
+      criteria.columnNames.every((columnName) => {
+        const value = argsData[columnName] as string | undefined;
+
+        return (
+          !!value && value.length >= settings.minLengthOfStringForDuplicateCheck
+        );
+      }),
+    );
+
+    const filterCriteria = criteriaWithMatchingArgs.map((criteria) =>
+      Object.fromEntries(
+        criteria.columnNames.map((columnName) => [
+          columnName,
+          { eq: argsData[columnName] },
+        ]),
+      ),
+    );
+
+    return {
+      // when filtering by an existing record, we need to filter that explicit record out
+      ...(filteringByExistingRecordId && {
+        id: { neq: filteringByExistingRecordId },
+      }),
+      // keep condition as "or" to get results by more duplicate criteria
+      or: filterCriteria,
+    };
+  }
+
+  private getApplicableDuplicateCriteriaCollection(
+    objectMetadataItem: ObjectMetadataInterface,
+  ) {
+    return DUPLICATE_CRITERIA_COLLECTION.filter(
+      (duplicateCriteria) =>
+        duplicateCriteria.objectName === objectMetadataItem.nameSingular,
+    );
+  }
+
+  /**
+   * TODO: Remove this code by September 1st, 2024 if it isn't used
+   * It was build to be used by the upsertMany function, but it was not used.
+   * It's a re-implementation of the methods to findDuplicates, but done
+   * at the SQL layer instead of doing it at the GraphQL layer
+   * 
+  async findDuplicate(
+    data: Partial<Record>,
+    objectMetadata: ObjectMetadataInterface,
+    workspaceId: string,
+  ) {
+    const dataSourceSchema =
+      this.workspaceDataSourceService.getSchemaName(workspaceId);
+
+    const { duplicateWhereClause, duplicateWhereParameters } =
+      this.buildDuplicateConditionForUpsert(objectMetadata, data);
+
+    const results = await this.workspaceDataSourceService.executeRawQuery(
+      `
+          SELECT 
+             *
+          FROM
+              ${dataSourceSchema}."${computeObjectTargetTable(
+                objectMetadata,
+              )}" p
+          WHERE
+              ${duplicateWhereClause}
+          `,
+      duplicateWhereParameters,
+      workspaceId,
+    );
+
+    return results.length > 0 ? results[0] : null;
+  }
+
+  private buildDuplicateConditionForUpsert(
+    objectMetadata: ObjectMetadataInterface,
+    data: Partial<Record>,
+  ) {
+    const criteriaCollection = this.getApplicableDuplicateCriteriaCollection(
+      objectMetadata,
+    ).filter(
+      (duplicateCriteria) => duplicateCriteria.useAsUniqueKeyForUpsert === true,
+    );
+
+    const whereClauses: string[] = [];
+    const whereParameters: any[] = [];
+    let parameterIndex = 1;
+
+    criteriaCollection.forEach((c) => {
+      const clauseParts: string[] = [];
+
+      c.columnNames.forEach((column) => {
+        const dataKey = Object.keys(data).find(
+          (key) => key.toLowerCase() === column.toLowerCase(),
+        );
+
+        if (dataKey) {
+          clauseParts.push(`p."${column}" = $${parameterIndex}`);
+          whereParameters.push(data[dataKey]);
+          parameterIndex++;
+        }
+      });
+      if (clauseParts.length > 0) {
+        whereClauses.push(`(${clauseParts.join(' AND ')})`);
+      }
+    });
+
+    const duplicateWhereClause = whereClauses.join(' OR ');
+    const duplicateWhereParameters = whereParameters;
+
+    return { duplicateWhereClause, duplicateWhereParameters };
+  }
+  *
+  */
+}