Aggregated queries #1 (#8345)

First step of https://github.com/twentyhq/twenty/issues/6868 Adds min.., max.. queries for DATETIME fields adds min.., max.., avg.., sum.. queries for NUMBER fields (count distinct operation and composite fields such as CURRENCY handling will be dealt with in a future PR) <img width="1422" alt="Capture d’écran 2024-11-06 à 15 48 46" src="https://github.com/user-attachments/assets/4bcdece0-ad3e-4536-9720-fe4044a36719"> --------- Co-authored-by: Charles Bochet <charles@twenty.com> Co-authored-by: Weiko <corentin@twenty.com>
2024-11-14 18:05:05 +01:00
parent c966533f26
commit a799370483
93 changed files with 1590 additions and 1178 deletions
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/tests/query-runner-args.factory.spec.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/tests/query-runner-args.factory.spec.ts
@ -1,12 +1,12 @@
 import { Test, TestingModule } from '@nestjs/testing';

 import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface';
-import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
 import { ResolverArgsType } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';

 import { QueryRunnerArgsFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory';
-import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
 import { RecordPositionFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/record-position.factory';
+import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';

 describe('QueryRunnerArgsFactory', () => {
  const recordPositionFactory = {
@ -14,13 +14,29 @@ describe('QueryRunnerArgsFactory', () => {
  };
  const workspaceId = 'workspaceId';
  const options = {
-    fieldMetadataCollection: [
-      { name: 'position', type: FieldMetadataType.POSITION },
-      { name: 'testNumber', type: FieldMetadataType.NUMBER },
-    ] as FieldMetadataInterface[],
-    objectMetadataItem: { isCustom: true, nameSingular: 'test' },
    authContext: { workspace: { id: workspaceId } },
-  } as WorkspaceQueryRunnerOptions;
+    objectMetadataItemWithFieldMaps: {
+      isCustom: true,
+      nameSingular: 'testNumber',
+      fieldsByName: {
+        position: {
+          type: FieldMetadataType.POSITION,
+          isCustom: true,
+          nameSingular: 'position',
+        },
+        testNumber: {
+          type: FieldMetadataType.NUMBER,
+          isCustom: true,
+          nameSingular: 'testNumber',
+        },
+        otherField: {
+          type: FieldMetadataType.TEXT,
+          isCustom: true,
+          nameSingular: 'otherField',
+        },
+      } as unknown as FieldMetadataMap,
+    },
+  } as unknown as WorkspaceQueryRunnerOptions;

  let factory: QueryRunnerArgsFactory;

@ -61,7 +77,7 @@ describe('QueryRunnerArgsFactory', () => {
    it('createMany type should override data position and number', async () => {
      const args = {
        id: 'uuid',
-        data: [{ position: 'last', testNumber: '1' }],
+        data: [{ position: 'last', testNumber: 1 }],
      };

      const result = await factory.create(
@ -72,7 +88,7 @@ describe('QueryRunnerArgsFactory', () => {

      expect(recordPositionFactory.create).toHaveBeenCalledWith(
        'last',
-        { isCustom: true, nameSingular: 'test' },
+        { isCustom: true, nameSingular: 'testNumber' },
        workspaceId,
        0,
      );
@ -85,7 +101,7 @@ describe('QueryRunnerArgsFactory', () => {
    it('createMany type should override position if not present', async () => {
      const args = {
        id: 'uuid',
-        data: [{ testNumber: '1' }],
+        data: [{ testNumber: 1 }],
      };

      const result = await factory.create(
@ -96,7 +112,7 @@ describe('QueryRunnerArgsFactory', () => {

      expect(recordPositionFactory.create).toHaveBeenCalledWith(
        'first',
-        { isCustom: true, nameSingular: 'test' },
+        { isCustom: true, nameSingular: 'testNumber' },
        workspaceId,
        0,
      );
@ -109,7 +125,7 @@ describe('QueryRunnerArgsFactory', () => {
    it('findMany type should override data position and number', async () => {
      const args = {
        id: 'uuid',
-        filter: { testNumber: { eq: '1' }, otherField: { eq: 'test' } },
+        filter: { testNumber: { eq: 1 }, otherField: { eq: 'test' } },
      };

      const result = await factory.create(
@ -127,7 +143,7 @@ describe('QueryRunnerArgsFactory', () => {
    it('findOne type should override number in filter', async () => {
      const args = {
        id: 'uuid',
-        filter: { testNumber: { eq: '1' }, otherField: { eq: 'test' } },
+        filter: { testNumber: { eq: 1 }, otherField: { eq: 'test' } },
      };

      const result = await factory.create(
@ -143,23 +159,14 @@ describe('QueryRunnerArgsFactory', () => {
    });

    it('findDuplicates type should override number in data and id', async () => {
-      const optionsDuplicate = {
-        fieldMetadataCollection: [
-          { name: 'id', type: FieldMetadataType.NUMBER },
-          { name: 'testNumber', type: FieldMetadataType.NUMBER },
-        ] as FieldMetadataInterface[],
-        objectMetadataItem: { isCustom: true, nameSingular: 'test' },
-        authContext: { workspace: { id: workspaceId } },
-      } as WorkspaceQueryRunnerOptions;
-
      const args = {
-        ids: ['123'],
-        data: [{ testNumber: '1', otherField: 'test' }],
+        ids: [123],
+        data: [{ testNumber: 1, otherField: 'test' }],
      };

      const result = await factory.create(
        args,
-        optionsDuplicate,
+        options,
        ResolverArgsType.FindDuplicates,
      );

--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory.ts
@ -1,6 +1,9 @@
 import { Injectable } from '@nestjs/common';

-import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
+import {
+  ObjectRecord,
+  ObjectRecordFilter,
+} from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
 import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface';
 import {
  CreateManyResolverArgs,
@ -10,13 +13,11 @@ import {
  ResolverArgs,
  ResolverArgsType,
 } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
-import {
-  Record,
-  RecordFilter,
-} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';

 import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
 import { hasPositionField } from 'src/engine/metadata-modules/object-metadata/utils/has-position-field.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';

 import { RecordPositionFactory } from './record-position.factory';

@ -34,27 +35,28 @@ export class QueryRunnerArgsFactory {
    options: WorkspaceQueryRunnerOptions,
    resolverArgsType: ResolverArgsType,
  ) {
-    const fieldMetadataCollection = options.fieldMetadataCollection;
+    const fieldMetadataMapByNameByName =
+      options.objectMetadataItemWithFieldMaps.fieldsByName;

-    const fieldMetadataMap = new Map(
-      fieldMetadataCollection.map((fieldMetadata) => [
-        fieldMetadata.name,
-        fieldMetadata,
-      ]),
+    const shouldBackfillPosition = hasPositionField(
+      options.objectMetadataItemWithFieldMaps,
    );

-    const shouldBackfillPosition = hasPositionField(options.objectMetadataItem);
-
    switch (resolverArgsType) {
      case ResolverArgsType.CreateMany:
        return {
          ...args,
          data: await Promise.all(
            (args as CreateManyResolverArgs).data?.map((arg, index) =>
-              this.overrideDataByFieldMetadata(arg, options, fieldMetadataMap, {
-                argIndex: index,
-                shouldBackfillPosition,
-              }),
+              this.overrideDataByFieldMetadata(
+                arg,
+                options,
+                fieldMetadataMapByNameByName,
+                {
+                  argIndex: index,
+                  shouldBackfillPosition,
+                },
+              ),
            ) ?? [],
          ),
        } satisfies CreateManyResolverArgs;
@ -63,7 +65,7 @@ export class QueryRunnerArgsFactory {
          ...args,
          filter: await this.overrideFilterByFieldMetadata(
            (args as FindOneResolverArgs).filter,
-            fieldMetadataMap,
+            fieldMetadataMapByNameByName,
          ),
        };
      case ResolverArgsType.FindMany:
@ -71,7 +73,7 @@ export class QueryRunnerArgsFactory {
          ...args,
          filter: await this.overrideFilterByFieldMetadata(
            (args as FindManyResolverArgs).filter,
-            fieldMetadataMap,
+            fieldMetadataMapByNameByName,
          ),
        };

@ -80,15 +82,24 @@ export class QueryRunnerArgsFactory {
          ...args,
          ids: (await Promise.all(
            (args as FindDuplicatesResolverArgs).ids?.map((id) =>
-              this.overrideValueByFieldMetadata('id', id, fieldMetadataMap),
+              this.overrideValueByFieldMetadata(
+                'id',
+                id,
+                fieldMetadataMapByNameByName,
+              ),
            ) ?? [],
          )) as string[],
          data: await Promise.all(
            (args as FindDuplicatesResolverArgs).data?.map((arg, index) =>
-              this.overrideDataByFieldMetadata(arg, options, fieldMetadataMap, {
-                argIndex: index,
-                shouldBackfillPosition,
-              }),
+              this.overrideDataByFieldMetadata(
+                arg,
+                options,
+                fieldMetadataMapByNameByName,
+                {
+                  argIndex: index,
+                  shouldBackfillPosition,
+                },
+              ),
            ) ?? [],
          ),
        } satisfies FindDuplicatesResolverArgs;
@ -98,9 +109,9 @@ export class QueryRunnerArgsFactory {
  }

  private async overrideDataByFieldMetadata(
-    data: Partial<Record> | undefined,
+    data: Partial<ObjectRecord> | undefined,
    options: WorkspaceQueryRunnerOptions,
-    fieldMetadataMap: Map<string, FieldMetadataInterface>,
+    fieldMetadataMapByNameByName: Record<string, FieldMetadataInterface>,
    argPositionBackfillInput: ArgPositionBackfillInput,
  ) {
    if (!data) {
@ -111,7 +122,7 @@ export class QueryRunnerArgsFactory {

    const createArgPromiseByArgKey = Object.entries(data).map(
      async ([key, value]) => {
-        const fieldMetadata = fieldMetadataMap.get(key);
+        const fieldMetadata = fieldMetadataMapByNameByName[key];

        if (!fieldMetadata) {
          return [key, await Promise.resolve(value)];
@ -126,8 +137,9 @@ export class QueryRunnerArgsFactory {
              await this.recordPositionFactory.create(
                value,
                {
-                  isCustom: options.objectMetadataItem.isCustom,
-                  nameSingular: options.objectMetadataItem.nameSingular,
+                  isCustom: options.objectMetadataItemWithFieldMaps.isCustom,
+                  nameSingular:
+                    options.objectMetadataItemWithFieldMaps.nameSingular,
                },
                options.authContext.workspace.id,
                argPositionBackfillInput.argIndex,
@ -154,8 +166,9 @@ export class QueryRunnerArgsFactory {
          await this.recordPositionFactory.create(
            'first',
            {
-              isCustom: options.objectMetadataItem.isCustom,
-              nameSingular: options.objectMetadataItem.nameSingular,
+              isCustom: options.objectMetadataItemWithFieldMaps.isCustom,
+              nameSingular:
+                options.objectMetadataItemWithFieldMaps.nameSingular,
            },
            options.authContext.workspace.id,
            argPositionBackfillInput.argIndex,
@ -168,23 +181,27 @@ export class QueryRunnerArgsFactory {
  }

  private overrideFilterByFieldMetadata(
-    filter: RecordFilter | undefined,
-    fieldMetadataMap: Map<string, FieldMetadataInterface>,
+    filter: ObjectRecordFilter | undefined,
+    fieldMetadataMapByName: Record<string, FieldMetadataInterface>,
  ) {
    if (!filter) {
      return;
    }

-    const overrideFilter = (filterObject: RecordFilter) => {
+    const overrideFilter = (filterObject: ObjectRecordFilter) => {
      return Object.entries(filterObject).reduce((acc, [key, value]) => {
        if (key === 'and' || key === 'or') {
-          acc[key] = value.map((nestedFilter: RecordFilter) =>
+          acc[key] = value.map((nestedFilter: ObjectRecordFilter) =>
            overrideFilter(nestedFilter),
          );
        } else if (key === 'not') {
          acc[key] = overrideFilter(value);
        } else {
-          acc[key] = this.transformValueByType(key, value, fieldMetadataMap);
+          acc[key] = this.transformValueByType(
+            key,
+            value,
+            fieldMetadataMapByName,
+          );
        }

        return acc;
@ -197,9 +214,9 @@ export class QueryRunnerArgsFactory {
  private transformValueByType(
    key: string,
    value: any,
-    fieldMetadataMap: Map<string, FieldMetadataInterface>,
+    fieldMetadataMapByName: FieldMetadataMap,
  ) {
-    const fieldMetadata = fieldMetadataMap.get(key);
+    const fieldMetadata = fieldMetadataMapByName[key];

    if (!fieldMetadata) {
      return value;
@ -226,9 +243,9 @@ export class QueryRunnerArgsFactory {
  private async overrideValueByFieldMetadata(
    key: string,
    value: any,
-    fieldMetadataMap: Map<string, FieldMetadataInterface>,
+    fieldMetadataMapByName: FieldMetadataMap,
  ) {
-    const fieldMetadata = fieldMetadataMap.get(key);
+    const fieldMetadata = fieldMetadataMapByName[key];

    if (!fieldMetadata) {
      return value;
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/interfaces/pg-graphql.interface.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/interfaces/pg-graphql.interface.ts
@ -1,15 +0,0 @@
-import { Record as IRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
-
-export interface PGGraphQLResponse<Data = any> {
-  resolve: {
-    data: Data;
-    errors: any[];
-  };
-}
-
-export type PGGraphQLResult<Data = any> = [PGGraphQLResponse<Data>];
-
-export interface PGGraphQLMutation<Record = IRecord> {
-  affectedRows: number;
-  records: Record[];
-}
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface.ts
@ -1,20 +1,12 @@
 import { GraphQLResolveInfo } from 'graphql';

-import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
-import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
-
 import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
-import {
-  ObjectMetadataMap,
-  ObjectMetadataMapItem,
-} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';

 export interface WorkspaceQueryRunnerOptions {
  authContext: AuthContext;
  info: GraphQLResolveInfo;
-  objectMetadataItem: ObjectMetadataInterface;
-  fieldMetadataCollection: FieldMetadataInterface[];
-  objectMetadataCollection: ObjectMetadataInterface[];
-  objectMetadataMap: ObjectMetadataMap;
-  objectMetadataMapItem: ObjectMetadataMapItem;
+  objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps;
+  objectMetadataMaps: ObjectMetadataMaps;
 }
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts
@ -1,12 +1,12 @@
 import { Injectable } from '@nestjs/common';
 import { OnEvent } from '@nestjs/event-emitter';

+import { OnDatabaseEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-event.decorator';
+import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
 import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service';
 import { ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
 import { TelemetryService } from 'src/engine/core-modules/telemetry/telemetry.service';
 import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
-import { OnDatabaseEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-event.decorator';
-import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';

@Injectable()
 export class TelemetryListener {
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/utils/with-soft-deleted.util.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/utils/with-soft-deleted.util.ts
@ -1,8 +1,8 @@
-import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';

 import { isDefined } from 'src/utils/is-defined';

-export const withSoftDeleted = <T extends RecordFilter>(
+export const withSoftDeleted = <T extends ObjectRecordFilter>(
  filter: T | undefined | null,
 ): boolean => {
  if (!isDefined(filter)) {
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/utils/workspace-query-runner-graphql-api-exception-handler.util.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/utils/workspace-query-runner-graphql-api-exception-handler.util.ts
@ -31,22 +31,23 @@ export const workspaceQueryRunnerGraphqlApiExceptionHandler = (
      if (indexNameMatch) {
        const indexName = indexNameMatch[1];

-        const deletedAtFieldMetadata = context.objectMetadataItem.fields.find(
-          (field) => field.name === 'deletedAt',
-        );
+        const deletedAtFieldMetadata =
+          context.objectMetadataItemWithFieldMaps.fieldsByName['deletedAt'];

-        const affectedColumns = context.objectMetadataItem.indexMetadatas
-          .find((index) => index.name === indexName)
-          ?.indexFieldMetadatas?.filter(
-            (field) => field.fieldMetadataId !== deletedAtFieldMetadata?.id,
-          )
-          .map((indexField) => {
-            const fieldMetadata = context.objectMetadataItem.fields.find(
-              (objectField) => indexField.fieldMetadataId === objectField.id,
-            );
+        const affectedColumns =
+          context.objectMetadataItemWithFieldMaps.indexMetadatas
+            .find((index) => index.name === indexName)
+            ?.indexFieldMetadatas?.filter(
+              (field) => field.fieldMetadataId !== deletedAtFieldMetadata?.id,
+            )
+            .map((indexField) => {
+              const fieldMetadata =
+                context.objectMetadataItemWithFieldMaps.fieldsById[
+                  indexField.fieldMetadataId
+                ];

-            return fieldMetadata?.label;
-          });
+              return fieldMetadata?.label;
+            });

        const columnNames = affectedColumns?.join(', ');

--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-hook/workspace-query-hook.service.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-hook/workspace-query-hook.service.ts
@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';

 import merge from 'lodash.merge';

-import { Record as IRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
+import { ObjectRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
 import { WorkspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';

 import { WorkspaceQueryHookKey } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-hook/decorators/workspace-query-hook.decorator';
@ -53,13 +53,13 @@ export class WorkspaceQueryHookService {

  public async executePostQueryHooks<
    T extends WorkspaceResolverBuilderMethodNames,
-    Record extends IRecord = IRecord,
+    U extends ObjectRecord = ObjectRecord,
  >(
    authContext: AuthContext,
    // TODO: We should allow wildcard for object name
    objectName: string,
    methodName: T,
-    payload: Record[],
+    payload: U[],
  ): Promise<void> {
    const key: WorkspaceQueryHookKey = `${objectName}.${methodName}`;
    const postHookInstances =