Aggregated queries #1 (#8345)

First step of https://github.com/twentyhq/twenty/issues/6868 Adds min.., max.. queries for DATETIME fields adds min.., max.., avg.., sum.. queries for NUMBER fields (count distinct operation and composite fields such as CURRENCY handling will be dealt with in a future PR) <img width="1422" alt="Capture d’écran 2024-11-06 à 15 48 46" src="https://github.com/user-attachments/assets/4bcdece0-ad3e-4536-9720-fe4044a36719"> --------- Co-authored-by: Charles Bochet <charles@twenty.com> Co-authored-by: Weiko <corentin@twenty.com>
2024-11-14 18:05:05 +01:00
parent c966533f26
commit a799370483
93 changed files with 1590 additions and 1178 deletions
--- a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-column.factory.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-column.factory.ts
@ -10,7 +10,7 @@ import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-me
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
 import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
 import { serializeDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-default-value';
-import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
 import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
 import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';

@ -20,10 +20,10 @@ type EntitySchemaColumnMap = {

@Injectable()
 export class EntitySchemaColumnFactory {
-  create(fieldMetadataMap: FieldMetadataMap): EntitySchemaColumnMap {
+  create(fieldMetadataMapByName: FieldMetadataMap): EntitySchemaColumnMap {
    let entitySchemaColumnMap: EntitySchemaColumnMap = {};

-    const fieldMetadataCollection = Object.values(fieldMetadataMap);
+    const fieldMetadataCollection = Object.values(fieldMetadataMapByName);

    for (const fieldMetadata of fieldMetadataCollection) {
      const key = fieldMetadata.name;
--- a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts
@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';

 import { EntitySchemaRelationOptions } from 'typeorm';

-import {
-  FieldMetadataMap,
-  ObjectMetadataMap,
-} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { FieldMetadataMap } from 'src/engine/metadata-modules/types/field-metadata-map';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { determineRelationDetails } from 'src/engine/twenty-orm/utils/determine-relation-details.util';
 import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';

@ -18,12 +16,12 @@ export class EntitySchemaRelationFactory {
  constructor() {}

  async create(
-    fieldMetadataMap: FieldMetadataMap,
-    objectMetadataMap: ObjectMetadataMap,
+    fieldMetadataMapByName: FieldMetadataMap,
+    objectMetadataMaps: ObjectMetadataMaps,
  ): Promise<EntitySchemaRelationMap> {
    const entitySchemaRelationMap: EntitySchemaRelationMap = {};

-    const fieldMetadataCollection = Object.values(fieldMetadataMap);
+    const fieldMetadataCollection = Object.values(fieldMetadataMapByName);

    for (const fieldMetadata of fieldMetadataCollection) {
      if (!isRelationFieldMetadataType(fieldMetadata.type)) {
@ -42,7 +40,7 @@ export class EntitySchemaRelationFactory {
      const relationDetails = await determineRelationDetails(
        fieldMetadata,
        relationMetadata,
-        objectMetadataMap,
+        objectMetadataMaps,
      );

      entitySchemaRelationMap[fieldMetadata.name] = {
--- a/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts
@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common';

 import { EntitySchema } from 'typeorm';

-import {
-  ObjectMetadataMap,
-  ObjectMetadataMapItem,
-} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
 import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
 import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage';
@ -20,17 +18,17 @@ export class EntitySchemaFactory {

  async create(
    workspaceId: string,
-    metadataVersion: number,
-    objectMetadata: ObjectMetadataMapItem,
-    objectMetadataMap: ObjectMetadataMap,
+    _metadataVersion: number,
+    objectMetadata: ObjectMetadataItemWithFieldMaps,
+    objectMetadataMaps: ObjectMetadataMaps,
  ): Promise<EntitySchema> {
    const columns = this.entitySchemaColumnFactory.create(
-      objectMetadata.fields,
+      objectMetadata.fieldsByName,
    );

    const relations = await this.entitySchemaRelationFactory.create(
-      objectMetadata.fields,
-      objectMetadataMap,
+      objectMetadata.fieldsByName,
+      objectMetadataMaps,
    );

    const entitySchema = new EntitySchema({
--- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
@ -86,13 +86,13 @@ export class WorkspaceDatasourceFactory {

            let cachedEntitySchemas: EntitySchema[];

-            const cachedObjectMetadataMap =
-              await this.workspaceCacheStorageService.getObjectMetadataMap(
+            const cachedObjectMetadataMaps =
+              await this.workspaceCacheStorageService.getObjectMetadataMaps(
                workspaceId,
                cachedWorkspaceMetadataVersion,
              );

-            if (!cachedObjectMetadataMap) {
+            if (!cachedObjectMetadataMaps) {
              throw new TwentyORMException(
                `Workspace Schema not found for workspace ${workspaceId}`,
                TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND,
@ -105,13 +105,14 @@ export class WorkspaceDatasourceFactory {
              );
            } else {
              const entitySchemas = await Promise.all(
-                Object.values(cachedObjectMetadataMap).map((objectMetadata) =>
-                  this.entitySchemaFactory.create(
-                    workspaceId,
-                    cachedWorkspaceMetadataVersion,
-                    objectMetadata,
-                    cachedObjectMetadataMap,
-                  ),
+                Object.values(cachedObjectMetadataMaps.byId).map(
+                  (objectMetadata) =>
+                    this.entitySchemaFactory.create(
+                      workspaceId,
+                      cachedWorkspaceMetadataVersion,
+                      objectMetadata,
+                      cachedObjectMetadataMaps,
+                    ),
                ),
              );

@ -127,7 +128,7 @@ export class WorkspaceDatasourceFactory {
            const workspaceDataSource = new WorkspaceDataSource(
              {
                workspaceId,
-                objectMetadataMap: cachedObjectMetadataMap,
+                objectMetadataMaps: cachedObjectMetadataMaps,
              },
              {
                url:
--- a/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts
@ -1,6 +1,6 @@
-import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';

 export interface WorkspaceInternalContext {
  workspaceId: string;
-  objectMetadataMap: ObjectMetadataMap;
+  objectMetadataMaps: ObjectMetadataMaps;
 }
--- a/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts
@ -22,7 +22,7 @@ import { UpsertOptions } from 'typeorm/repository/UpsertOptions';

 import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/workspace-internal-context.interface';

-import { ObjectMetadataMapItem } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage';
 import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
 import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
@ -631,13 +631,15 @@ export class WorkspaceRepository<
    }

    const objectMetadata =
-      this.internalContext.objectMetadataMap[objectMetadataName];
+      this.internalContext.objectMetadataMaps.byNameSingular[
+        objectMetadataName
+      ];

    if (!objectMetadata) {
      throw new Error(
        `Object metadata for object "${objectMetadataName}" is missing ` +
          `in workspace "${this.internalContext.workspaceId}" ` +
-          `with object metadata collection length: ${this.internalContext.objectMetadataMap.length}`,
+          `with object metadata collection length: ${this.internalContext.objectMetadataMaps.byNameSingular.length}`,
      );
    }

@ -666,12 +668,12 @@ export class WorkspaceRepository<

  async formatResult<T>(
    data: T,
-    objectMetadata?: ObjectMetadataMapItem,
+    objectMetadata?: ObjectMetadataItemWithFieldMaps,
  ): Promise<T> {
    objectMetadata ??= await this.getObjectMetadataFromTarget();

-    const objectMetadataMap = this.internalContext.objectMetadataMap;
+    const objectMetadataMaps = this.internalContext.objectMetadataMaps;

-    return formatResult(data, objectMetadata, objectMetadataMap) as T;
+    return formatResult(data, objectMetadata, objectMetadataMaps) as T;
  }
 }
--- a/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts
@ -3,7 +3,7 @@ import { RelationType } from 'typeorm/metadata/types/RelationTypes';
 import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';

 import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
-import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { computeRelationType } from 'src/engine/twenty-orm/utils/compute-relation-type.util';

 interface RelationDetails {
@ -16,22 +16,25 @@ interface RelationDetails {
 export async function determineRelationDetails(
  fieldMetadata: FieldMetadataInterface,
  relationMetadata: RelationMetadataEntity,
-  objectMetadataMap: ObjectMetadataMap,
+  objectMetadataMaps: ObjectMetadataMaps,
 ): Promise<RelationDetails> {
  const relationType = computeRelationType(fieldMetadata, relationMetadata);
-  const fromObjectMetadata = objectMetadataMap[fieldMetadata.objectMetadataId];
-  let toObjectMetadata = objectMetadataMap[relationMetadata.toObjectMetadataId];
+  const fromObjectMetadata =
+    objectMetadataMaps.byId[fieldMetadata.objectMetadataId];
+  let toObjectMetadata =
+    objectMetadataMaps.byId[relationMetadata.toObjectMetadataId];

  // RelationMetadata always store the relation from the perspective of the `from` object, MANY_TO_ONE relations are not stored yet
  if (relationType === 'many-to-one') {
-    toObjectMetadata = objectMetadataMap[relationMetadata.fromObjectMetadataId];
+    toObjectMetadata =
+      objectMetadataMaps.byId[relationMetadata.fromObjectMetadataId];
  }

  if (!fromObjectMetadata || !toObjectMetadata) {
    throw new Error('Object metadata not found');
  }

-  const toFieldMetadata = Object.values(toObjectMetadata.fields).find(
+  const toFieldMetadata = Object.values(toObjectMetadata.fieldsById).find(
    (field) =>
      relationType === 'many-to-one'
        ? field.id === relationMetadata.fromFieldMetadataId
--- a/packages/twenty-server/src/engine/twenty-orm/utils/format-data.util.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/utils/format-data.util.ts
@ -3,26 +3,28 @@ import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metada
 import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
 import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { ObjectMetadataMapItem } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
 import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
 import { capitalize } from 'src/utils/capitalize';

 export function formatData<T>(
  data: T,
-  objectMetadata: ObjectMetadataMapItem,
+  objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
 ): T {
  if (!data) {
    return data;
  }

  if (Array.isArray(data)) {
-    return data.map((item) => formatData(item, objectMetadata)) as T;
+    return data.map((item) =>
+      formatData(item, objectMetadataItemWithFieldMaps),
+    ) as T;
  }

  const newData: Record<string, any> = {};

  for (const [key, value] of Object.entries(data)) {
-    const fieldMetadata = objectMetadata.fields[key];
+    const fieldMetadata = objectMetadataItemWithFieldMaps.fieldsByName[key];

    if (!fieldMetadata) {
      throw new Error(
--- a/packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/utils/format-result.util.ts
@ -6,18 +6,16 @@ import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-meta
 import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
 import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
 import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
-import {
-  ObjectMetadataMap,
-  ObjectMetadataMapItem,
-} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
 import { computeRelationType } from 'src/engine/twenty-orm/utils/compute-relation-type.util';
 import { getCompositeFieldMetadataCollection } from 'src/engine/twenty-orm/utils/get-composite-field-metadata-collection';
 import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';

 export function formatResult<T>(
  data: T,
-  objectMetadata: ObjectMetadataMapItem,
-  objectMetadataMap: ObjectMetadataMap,
+  ObjectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
+  objectMetadataMaps: ObjectMetadataMaps,
 ): T {
  if (!data) {
    return data;
@ -25,7 +23,7 @@ export function formatResult<T>(

  if (Array.isArray(data)) {
    return data.map((item) =>
-      formatResult(item, objectMetadata, objectMetadataMap),
+      formatResult(item, ObjectMetadataItemWithFieldMaps, objectMetadataMaps),
    ) as T;
  }

@ -33,12 +31,13 @@ export function formatResult<T>(
    return data;
  }

-  if (!objectMetadata) {
+  if (!ObjectMetadataItemWithFieldMaps) {
    throw new Error('Object metadata is missing');
  }

-  const compositeFieldMetadataCollection =
-    getCompositeFieldMetadataCollection(objectMetadata);
+  const compositeFieldMetadataCollection = getCompositeFieldMetadataCollection(
+    ObjectMetadataItemWithFieldMaps,
+  );

  const compositeFieldMetadataMap = new Map(
    compositeFieldMetadataCollection.flatMap((fieldMetadata) => {
@ -58,7 +57,7 @@ export function formatResult<T>(
  );

  const relationMetadataMap = new Map(
-    Object.values(objectMetadata.fields)
+    Object.values(ObjectMetadataItemWithFieldMaps.fieldsById)
      .filter(({ type }) => isRelationFieldMetadataType(type))
      .map((fieldMetadata) => [
        fieldMetadata.name,
@ -75,6 +74,8 @@ export function formatResult<T>(
      ]),
  );
  const newData: object = {};
+  const objectMetadaItemFieldsByName =
+    objectMetadataMaps.byId[ObjectMetadataItemWithFieldMaps.id]?.fieldsByName;

  for (const [key, value] of Object.entries(data)) {
    const compositePropertyArgs = compositeFieldMetadataMap.get(key);
@ -83,11 +84,15 @@ export function formatResult<T>(

    if (!compositePropertyArgs && !relationMetadata) {
      if (isPlainObject(value)) {
-        newData[key] = formatResult(value, objectMetadata, objectMetadataMap);
-      } else if (objectMetadata.fields[key]) {
+        newData[key] = formatResult(
+          value,
+          ObjectMetadataItemWithFieldMaps,
+          objectMetadataMaps,
+        );
+      } else if (objectMetadaItemFieldsByName[key]) {
        newData[key] = formatFieldMetadataValue(
          value,
-          objectMetadata.fields[key],
+          objectMetadaItemFieldsByName[key],
        );
      } else {
        newData[key] = value;
@ -98,10 +103,10 @@ export function formatResult<T>(

    if (relationMetadata) {
      const toObjectMetadata =
-        objectMetadataMap[relationMetadata.toObjectMetadataId];
+        objectMetadataMaps.byId[relationMetadata.toObjectMetadataId];

      const fromObjectMetadata =
-        objectMetadataMap[relationMetadata.fromObjectMetadataId];
+        objectMetadataMaps.byId[relationMetadata.fromObjectMetadataId];

      if (!toObjectMetadata) {
        throw new Error(
@ -118,7 +123,7 @@ export function formatResult<T>(
      newData[key] = formatResult(
        value,
        relationType === 'one-to-many' ? toObjectMetadata : fromObjectMetadata,
-        objectMetadataMap,
+        objectMetadataMaps,
      );
      continue;
    }
--- a/packages/twenty-server/src/engine/twenty-orm/utils/get-composite-field-metadata-collection.ts
+++ b/packages/twenty-server/src/engine/twenty-orm/utils/get-composite-field-metadata-collection.ts
@ -1,12 +1,14 @@
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { ObjectMetadataMapItem } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
+import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';

 export function getCompositeFieldMetadataCollection(
-  objectMetadata: ObjectMetadataMapItem,
+  ObjectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
 ) {
  const compositeFieldMetadataCollection = Object.values(
-    objectMetadata.fields,
-  ).filter((fieldMetadata) => isCompositeFieldMetadataType(fieldMetadata.type));
+    ObjectMetadataItemWithFieldMaps.fieldsById,
+  ).filter((fieldMetadataItem) =>
+    isCompositeFieldMetadataType(fieldMetadataItem.type),
+  );

  return compositeFieldMetadataCollection;
 }