feat: twenty orm for standard and custom objects (#6178)

### Overview

This PR builds upon #5153, adding the ability to get a repository for
custom objects. The `entitySchema` is now generated for both standard
and custom objects based on metadata stored in the database instead of
the decorated `WorkspaceEntity` in the code. This change ensures that
standard objects with custom fields and relations can also support
custom objects.

### Implementation Details

#### Key Changes:

- **Dynamic Schema Generation:** The `entitySchema` for standard and
custom objects is now dynamically generated from the metadata stored in
the database. This shift allows for greater flexibility and
adaptability, particularly for standard objects with custom fields and
relations.
  
- **Custom Object Repository Retrieval:** A repository for a custom
object can be retrieved using `TwentyORMManager` based on the object's
name. Here's an example of how this can be achieved:

  ```typescript
const repository = await this.twentyORMManager.getRepository('custom');
  /*
* `repository` variable will be typed as follows, ensuring that standard
fields and relations are properly typed:
   * const repository: WorkspaceRepository<CustomWorkspaceEntity & {
   *    [key: string]: any;
   * }>
   */
  const res = await repository.find({});
  ```

Fix #6179

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
Co-authored-by: Weiko <corentin@twenty.com>
This commit is contained in:
Jérémy M
2024-07-19 18:23:52 +02:00
committed by GitHub
parent a86cc5cb9c
commit 088d061b3e
33 changed files with 947 additions and 587 deletions

View File

@ -2,18 +2,17 @@ import { Injectable } from '@nestjs/common';
import { ColumnType, EntitySchemaColumnOptions } from 'typeorm';
import { WorkspaceFieldMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-field-metadata-args.interface';
import { WorkspaceRelationMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-relation-metadata-args.interface';
import { WorkspaceJoinColumnsMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-join-columns-metadata-args.interface';
import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
import { serializeDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-default-value';
import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { compositeTypeDefintions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { getJoinColumn } from 'src/engine/twenty-orm/utils/get-join-column.util';
import {
FieldMetadataEntity,
FieldMetadataType,
} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
type EntitySchemaColumnMap = {
[key: string]: EntitySchemaColumnOptions;
@ -22,17 +21,45 @@ type EntitySchemaColumnMap = {
@Injectable()
export class EntitySchemaColumnFactory {
create(
fieldMetadataArgsCollection: WorkspaceFieldMetadataArgs[],
relationMetadataArgsCollection: WorkspaceRelationMetadataArgs[],
joinColumnsMetadataArgsCollection: WorkspaceJoinColumnsMetadataArgs[],
workspaceId: string,
fieldMetadataCollection: FieldMetadataEntity[],
): EntitySchemaColumnMap {
let entitySchemaColumnMap: EntitySchemaColumnMap = {};
for (const fieldMetadataArgs of fieldMetadataArgsCollection) {
const key = fieldMetadataArgs.name;
for (const fieldMetadata of fieldMetadataCollection) {
const key = fieldMetadata.name;
if (isCompositeFieldMetadataType(fieldMetadataArgs.type)) {
const compositeColumns = this.createCompositeColumns(fieldMetadataArgs);
if (isRelationFieldMetadataType(fieldMetadata.type)) {
const relationMetadata =
fieldMetadata.fromRelationMetadata ??
fieldMetadata.toRelationMetadata;
if (!relationMetadata) {
throw new Error(
`Relation metadata is missing for field ${fieldMetadata.name}`,
);
}
const joinColumnKey = fieldMetadata.name + 'Id';
const joinColumn = fieldMetadataCollection.find(
(field) => field.name === joinColumnKey,
)
? joinColumnKey
: null;
if (joinColumn) {
entitySchemaColumnMap[joinColumn] = {
name: joinColumn,
type: 'uuid',
nullable: fieldMetadata.isNullable,
};
}
continue;
}
if (isCompositeFieldMetadataType(fieldMetadata.type)) {
const compositeColumns = this.createCompositeColumns(fieldMetadata);
entitySchemaColumnMap = {
...entitySchemaColumnMap,
@ -42,39 +69,23 @@ export class EntitySchemaColumnFactory {
continue;
}
const columnType = fieldMetadataTypeToColumnType(fieldMetadataArgs.type);
const defaultValue = serializeDefaultValue(
fieldMetadataArgs.defaultValue,
);
const columnType = fieldMetadataTypeToColumnType(fieldMetadata.type);
const defaultValue = serializeDefaultValue(fieldMetadata.defaultValue);
entitySchemaColumnMap[key] = {
name: key,
type: columnType as ColumnType,
primary: fieldMetadataArgs.isPrimary,
nullable: fieldMetadataArgs.isNullable,
// TODO: We should double check that
primary: key === 'id',
nullable: fieldMetadata.isNullable,
createDate: key === 'createdAt',
updateDate: key === 'updatedAt',
array: fieldMetadataArgs.type === FieldMetadataType.MULTI_SELECT,
array: fieldMetadata.type === FieldMetadataType.MULTI_SELECT,
default: defaultValue,
};
for (const relationMetadataArgs of relationMetadataArgsCollection) {
const joinColumn = getJoinColumn(
joinColumnsMetadataArgsCollection,
relationMetadataArgs,
);
if (joinColumn) {
entitySchemaColumnMap[joinColumn] = {
name: joinColumn,
type: 'uuid',
nullable: relationMetadataArgs.isNullable,
};
}
}
if (isEnumFieldMetadataType(fieldMetadataArgs.type)) {
const values = fieldMetadataArgs.options?.map((option) => option.value);
if (isEnumFieldMetadataType(fieldMetadata.type)) {
const values = fieldMetadata.options?.map((option) => option.value);
if (values && values.length > 0) {
entitySchemaColumnMap[key].enum = values;
@ -86,25 +97,25 @@ export class EntitySchemaColumnFactory {
}
private createCompositeColumns(
fieldMetadataArgs: WorkspaceFieldMetadataArgs,
fieldMetadata: FieldMetadataEntity,
): EntitySchemaColumnMap {
const entitySchemaColumnMap: EntitySchemaColumnMap = {};
const compositeType = compositeTypeDefintions.get(fieldMetadataArgs.type);
const compositeType = compositeTypeDefintions.get(fieldMetadata.type);
if (!compositeType) {
throw new Error(
`Composite type ${fieldMetadataArgs.type} is not defined in compositeTypeDefintions`,
`Composite type ${fieldMetadata.type} is not defined in compositeTypeDefintions`,
);
}
for (const compositeProperty of compositeType.properties) {
const columnName = computeCompositeColumnName(
fieldMetadataArgs.name,
fieldMetadata.name,
compositeProperty,
);
const columnType = fieldMetadataTypeToColumnType(compositeProperty.type);
const defaultValue = serializeDefaultValue(
fieldMetadataArgs.defaultValue?.[compositeProperty.name],
fieldMetadata.defaultValue?.[compositeProperty.name],
);
entitySchemaColumnMap[columnName] = {

View File

@ -1,14 +1,11 @@
import { Injectable } from '@nestjs/common';
import { EntitySchemaRelationOptions } from 'typeorm';
import { RelationType } from 'typeorm/metadata/types/RelationTypes';
import { WorkspaceRelationMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-relation-metadata-args.interface';
import { WorkspaceJoinColumnsMetadataArgs } from 'src/engine/twenty-orm/interfaces/workspace-join-columns-metadata-args.interface';
import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/convert-class-to-object-metadata-name.util';
import { RelationMetadataType } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { getJoinColumn } from 'src/engine/twenty-orm/utils/get-join-column.util';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
import { determineRelationDetails } from 'src/engine/twenty-orm/utils/determine-relation-details.util';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
type EntitySchemaRelationMap = {
[key: string]: EntitySchemaRelationOptions;
@ -16,55 +13,45 @@ type EntitySchemaRelationMap = {
@Injectable()
export class EntitySchemaRelationFactory {
create(
// eslint-disable-next-line @typescript-eslint/ban-types
target: Function,
relationMetadataArgsCollection: WorkspaceRelationMetadataArgs[],
joinColumnsMetadataArgsCollection: WorkspaceJoinColumnsMetadataArgs[],
): EntitySchemaRelationMap {
constructor(
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
async create(
workspaceId: string,
fieldMetadataCollection: FieldMetadataEntity[],
): Promise<EntitySchemaRelationMap> {
const entitySchemaRelationMap: EntitySchemaRelationMap = {};
for (const relationMetadataArgs of relationMetadataArgsCollection) {
const objectName = convertClassNameToObjectMetadataName(target.name);
const oppositeTarget = relationMetadataArgs.inverseSideTarget();
const oppositeObjectName = convertClassNameToObjectMetadataName(
oppositeTarget.name,
);
const relationType = this.getRelationType(relationMetadataArgs);
const joinColumn = getJoinColumn(
joinColumnsMetadataArgsCollection,
relationMetadataArgs,
for (const fieldMetadata of fieldMetadataCollection) {
if (!isRelationFieldMetadataType(fieldMetadata.type)) {
continue;
}
const relationMetadata =
fieldMetadata.fromRelationMetadata ?? fieldMetadata.toRelationMetadata;
if (!relationMetadata) {
throw new Error(
`Relation metadata is missing for field ${fieldMetadata.name}`,
);
}
const relationDetails = await determineRelationDetails(
workspaceId,
fieldMetadata,
relationMetadata,
this.workspaceCacheStorageService,
);
entitySchemaRelationMap[relationMetadataArgs.name] = {
type: relationType,
target: oppositeObjectName,
inverseSide: relationMetadataArgs.inverseSideFieldKey ?? objectName,
joinColumn: joinColumn
? {
name: joinColumn,
}
: undefined,
entitySchemaRelationMap[fieldMetadata.name] = {
type: relationDetails.relationType,
target: relationDetails.target,
inverseSide: relationDetails.inverseSide,
joinColumn: relationDetails.joinColumn,
};
}
return entitySchemaRelationMap;
}
private getRelationType(
relationMetadataArgs: WorkspaceRelationMetadataArgs,
): RelationType {
switch (relationMetadataArgs.type) {
case RelationMetadataType.ONE_TO_MANY:
return 'one-to-many';
case RelationMetadataType.MANY_TO_ONE:
return 'many-to-one';
case RelationMetadataType.ONE_TO_ONE:
return 'one-to-one';
case RelationMetadataType.MANY_TO_MANY:
return 'many-to-many';
default:
throw new Error('Invalid relation type');
}
}
}

View File

@ -1,53 +1,79 @@
import { Injectable, Type } from '@nestjs/common';
import { Injectable } from '@nestjs/common';
import { EntitySchema } from 'typeorm';
import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
import { metadataArgsStorage } from 'src/engine/twenty-orm/storage/metadata-args.storage';
import { ObjectLiteralStorage } from 'src/engine/twenty-orm/storage/object-literal.storage';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class EntitySchemaFactory {
constructor(
private readonly entitySchemaColumnFactory: EntitySchemaColumnFactory,
private readonly entitySchemaRelationFactory: EntitySchemaRelationFactory,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
create<T>(target: Type<T>): EntitySchema {
const entityMetadataArgs = metadataArgsStorage.filterEntities(target);
async create(
workspaceId: string,
objectMetadata: ObjectMetadataEntity,
): Promise<EntitySchema>;
if (!entityMetadataArgs) {
throw new Error('Entity metadata args are missing on this target');
async create(
workspaceId: string,
objectMetadataName: string,
): Promise<EntitySchema>;
async create(
workspaceId: string,
objectMetadataOrObjectMetadataName: ObjectMetadataEntity | string,
): Promise<EntitySchema> {
let objectMetadata: ObjectMetadataEntity | null =
typeof objectMetadataOrObjectMetadataName !== 'string'
? objectMetadataOrObjectMetadataName
: null;
if (typeof objectMetadataOrObjectMetadataName === 'string') {
objectMetadata =
(await this.workspaceCacheStorageService.getObjectMetadata(
workspaceId,
(objectMetadata) =>
objectMetadata.nameSingular === objectMetadataOrObjectMetadataName,
)) ?? null;
}
const fieldMetadataArgsCollection =
metadataArgsStorage.filterFields(target);
const joinColumnsMetadataArgsCollection =
metadataArgsStorage.filterJoinColumns(target);
const relationMetadataArgsCollection =
metadataArgsStorage.filterRelations(target);
if (!objectMetadata) {
throw new Error('Object metadata not found');
}
const columns = this.entitySchemaColumnFactory.create(
fieldMetadataArgsCollection,
relationMetadataArgsCollection,
joinColumnsMetadataArgsCollection,
workspaceId,
objectMetadata.fields,
);
const relations = this.entitySchemaRelationFactory.create(
target,
relationMetadataArgsCollection,
joinColumnsMetadataArgsCollection,
const relations = await this.entitySchemaRelationFactory.create(
workspaceId,
objectMetadata.fields,
);
const entitySchema = new EntitySchema({
name: entityMetadataArgs.nameSingular,
tableName: entityMetadataArgs.nameSingular,
name: objectMetadata.nameSingular,
tableName: computeTableName(
objectMetadata.nameSingular,
objectMetadata.isCustom,
),
columns,
relations,
});
ObjectLiteralStorage.setObjectLiteral(entitySchema, target);
WorkspaceEntitiesStorage.setEntitySchema(
workspaceId,
objectMetadata.nameSingular,
entitySchema,
);
return entitySchema;
}

View File

@ -1,7 +1,7 @@
import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory';
import { ScopedWorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-datasource.factory';
import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
export const entitySchemaFactories = [
@ -9,5 +9,5 @@ export const entitySchemaFactories = [
EntitySchemaRelationFactory,
EntitySchemaFactory,
WorkspaceDatasourceFactory,
ScopedWorkspaceDatasourceFactory,
ScopedWorkspaceContextFactory,
];

View File

@ -0,0 +1,26 @@
import { Inject, Injectable, Optional, Scope } from '@nestjs/common';
import { REQUEST } from '@nestjs/core';
@Injectable({ scope: Scope.REQUEST })
export class ScopedWorkspaceContextFactory {
constructor(
@Optional()
@Inject(REQUEST)
private readonly request: Request | null,
) {}
public create(): {
workspaceId: string | null;
cacheVersion: string | null;
} {
const workspaceId: string | undefined =
this.request?.['req']?.['workspaceId'];
const cacheVersion: string | undefined =
this.request?.['req']?.['cacheVersion'];
return {
workspaceId: workspaceId ?? null,
cacheVersion: cacheVersion ?? null,
};
}
}

View File

@ -1,27 +0,0 @@
import { Inject, Injectable, Optional, Scope } from '@nestjs/common';
import { REQUEST } from '@nestjs/core';
import { EntitySchema } from 'typeorm';
import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
@Injectable({ scope: Scope.REQUEST })
export class ScopedWorkspaceDatasourceFactory {
constructor(
@Optional()
@Inject(REQUEST)
private readonly request: Request | null,
private readonly workspaceDataSourceFactory: WorkspaceDatasourceFactory,
) {}
public async create(entities: EntitySchema[]) {
const workspaceId: string | undefined =
this.request?.['req']?.['workspaceId'];
if (!workspaceId) {
return null;
}
return this.workspaceDataSourceFactory.create(entities, workspaceId);
}
}

View File

@ -4,27 +4,21 @@ import { EntitySchema } from 'typeorm';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { DataSourceStorage } from 'src/engine/twenty-orm/storage/data-source.storage';
import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceDatasourceFactory {
constructor(
private readonly dataSourceService: DataSourceService,
private readonly environmentService: EnvironmentService,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
public async create(
entities: EntitySchema[],
workspaceId: string,
): Promise<WorkspaceDataSource | null> {
const storedWorkspaceDataSource =
DataSourceStorage.getDataSource(workspaceId);
if (storedWorkspaceDataSource) {
return storedWorkspaceDataSource;
}
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
workspaceId,
@ -34,27 +28,31 @@ export class WorkspaceDatasourceFactory {
return null;
}
const workspaceDataSource = new WorkspaceDataSource({
url:
dataSourceMetadata.url ??
this.environmentService.get('PG_DATABASE_URL'),
type: 'postgres',
logging: this.environmentService.get('DEBUG_MODE')
? ['query', 'error']
: ['error'],
schema: dataSourceMetadata.schema,
entities,
ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED')
? {
rejectUnauthorized: false,
}
: undefined,
});
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
workspaceCacheStorage: this.workspaceCacheStorageService,
},
{
url:
dataSourceMetadata.url ??
this.environmentService.get('PG_DATABASE_URL'),
type: 'postgres',
logging: this.environmentService.get('DEBUG_MODE')
? ['query', 'error']
: ['error'],
schema: dataSourceMetadata.schema,
entities,
ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED')
? {
rejectUnauthorized: false,
}
: undefined,
},
);
await workspaceDataSource.initialize();
DataSourceStorage.setDataSource(workspaceId, workspaceDataSource);
return workspaceDataSource;
}
}