admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Optimize metadata queries (#7013)

Browse Source 
In this PR:

1. Refactor guards to avoid duplicated queries: WorkspaceAuthGuard and
UserAuthGuard only check for existence of workspace and user in the
request without querying the database
This commit is contained in:
Charles Bochet
2024-09-13 19:11:32 +02:00
committed by Charles Bochet
parent cf8b1161cc
commit 523df5398a
132 changed files with 818 additions and 6372 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-column.factory.ts
View File

@ -2,15 +2,15 @@ import { Injectable } from '@nestjs/common';
import { ColumnType, EntitySchemaColumnOptions } from 'typeorm';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import {
FieldMetadataEntity,
FieldMetadataType,
} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { isEnumFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-enum-field-metadata-type.util';
import { serializeDefaultValue } from 'src/engine/metadata-modules/field-metadata/utils/serialize-default-value';
import { FieldMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { fieldMetadataTypeToColumnType } from 'src/engine/metadata-modules/workspace-migration/utils/field-metadata-type-to-column-type.util';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
@ -21,11 +21,13 @@ type EntitySchemaColumnMap = {
@Injectable()
export class EntitySchemaColumnFactory {
create(
fieldMetadataCollection: FieldMetadataEntity[],
fieldMetadataMap: FieldMetadataMap,
softDelete: boolean,
): EntitySchemaColumnMap {
let entitySchemaColumnMap: EntitySchemaColumnMap = {};
const fieldMetadataCollection = Object.values(fieldMetadataMap);
for (const fieldMetadata of fieldMetadataCollection) {
const key = fieldMetadata.name;
@ -102,7 +104,7 @@ export class EntitySchemaColumnFactory {
}
private createCompositeColumns(
fieldMetadata: FieldMetadataEntity,
fieldMetadata: FieldMetadataInterface,
): EntitySchemaColumnMap {
const entitySchemaColumnMap: EntitySchemaColumnMap = {};
const compositeType = compositeTypeDefinitions.get(fieldMetadata.type);

29
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema-relation.factory.ts
View File

@ -2,10 +2,12 @@ import { Injectable } from '@nestjs/common';
import { EntitySchemaRelationOptions } from 'typeorm';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import {
FieldMetadataMap,
ObjectMetadataMap,
} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { determineRelationDetails } from 'src/engine/twenty-orm/utils/determine-relation-details.util';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
type EntitySchemaRelationMap = {
[key: string]: EntitySchemaRelationOptions;
@ -13,17 +15,16 @@ type EntitySchemaRelationMap = {
@Injectable()
export class EntitySchemaRelationFactory {
constructor(
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
) {}
constructor() {}
async create(
workspaceId: string,
metadataVersion: number,
fieldMetadataCollection: FieldMetadataEntity[],
fieldMetadataMap: FieldMetadataMap,
objectMetadataMap: ObjectMetadataMap,
): Promise<EntitySchemaRelationMap> {
const entitySchemaRelationMap: EntitySchemaRelationMap = {};
const fieldMetadataCollection = Object.values(fieldMetadataMap);
for (const fieldMetadata of fieldMetadataCollection) {
if (!isRelationFieldMetadataType(fieldMetadata.type)) {
continue;
@ -38,20 +39,10 @@ export class EntitySchemaRelationFactory {
);
}
const objectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
workspaceId,
metadataVersion,
);
if (!objectMetadataCollection) {
throw new Error('Object metadata collection not found');
}
const relationDetails = await determineRelationDetails(
fieldMetadata,
relationMetadata,
objectMetadataCollection,
objectMetadataMap,
);
entitySchemaRelationMap[fieldMetadata.name] = {

11
packages/twenty-server/src/engine/twenty-orm/factories/entity-schema.factory.ts
View File

@ -2,7 +2,10 @@ import { Injectable } from '@nestjs/common';
import { EntitySchema } from 'typeorm';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import {
ObjectMetadataMap,
ObjectMetadataMapItem,
} from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory';
import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory';
import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage';
@ -18,7 +21,8 @@ export class EntitySchemaFactory {
async create(
workspaceId: string,
metadataVersion: number,
objectMetadata: ObjectMetadataEntity,
objectMetadata: ObjectMetadataMapItem,
objectMetadataMap: ObjectMetadataMap,
): Promise<EntitySchema> {
const columns = this.entitySchemaColumnFactory.create(
objectMetadata.fields,
@ -26,9 +30,8 @@ export class EntitySchemaFactory {
);
const relations = await this.entitySchemaRelationFactory.create(
workspaceId,
metadataVersion,
objectMetadata.fields,
objectMetadataMap,
);
const entitySchema = new EntitySchema({

13
packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
View File

@ -56,20 +56,20 @@ export class WorkspaceDatasourceFactory {
const workspaceDataSource = await this.cacheManager.execute(
`${workspaceId}-${desiredWorkspaceMetadataVersion}`,
async () => {
const cachedObjectMetadataCollection =
await this.workspaceCacheStorageService.getObjectMetadataCollection(
const cachedObjectMetadataMap =
await this.workspaceCacheStorageService.getObjectMetadataMap(
workspaceId,
desiredWorkspaceMetadataVersion,
);
if (!cachedObjectMetadataCollection) {
if (!cachedObjectMetadataMap) {
await this.workspaceMetadataCacheService.recomputeMetadataCache(
workspaceId,
true,
);
throw new TwentyORMException(
`Object metadata collection not found for workspace ${workspaceId}`,
`Object metadata map not found for workspace ${workspaceId}`,
TwentyORMExceptionCode.METADATA_COLLECTION_NOT_FOUND,
);
}
@ -100,11 +100,12 @@ export class WorkspaceDatasourceFactory {
);
} else {
const entitySchemas = await Promise.all(
cachedObjectMetadataCollection.map((objectMetadata) =>
Object.values(cachedObjectMetadataMap).map((objectMetadata) =>
this.entitySchemaFactory.create(
workspaceId,
desiredWorkspaceMetadataVersion,
objectMetadata,
cachedObjectMetadataMap,
),
),
);
@ -121,7 +122,7 @@ export class WorkspaceDatasourceFactory {
const workspaceDataSource = new WorkspaceDataSource(
{
workspaceId,
objectMetadataCollection: cachedObjectMetadataCollection,
objectMetadataMap: cachedObjectMetadataMap,
},
{
url:

4
packages/twenty-server/src/engine/twenty-orm/interfaces/workspace-internal-context.interface.ts
View File

@ -1,6 +1,6 @@
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
export interface WorkspaceInternalContext {
workspaceId: string;
objectMetadataCollection: ObjectMetadataEntity[];
objectMetadataMap: ObjectMetadataMap;
}

38
packages/twenty-server/src/engine/twenty-orm/repository/workspace.repository.ts
View File

@ -27,7 +27,8 @@ import { WorkspaceInternalContext } from 'src/engine/twenty-orm/interfaces/works
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { computeCompositeColumnName } from 'src/engine/metadata-modules/field-metadata/utils/compute-column-name.util';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { ObjectMetadataMapItem } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { WorkspaceEntitiesStorage } from 'src/engine/twenty-orm/storage/workspace-entities.storage';
import { computeRelationType } from 'src/engine/twenty-orm/utils/compute-relation-type.util';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
@ -623,15 +624,14 @@ export class WorkspaceRepository<
throw new Error('Object metadata name is missing');
}
const objectMetadata = this.internalContext.objectMetadataCollection.find(
(objectMetadata) => objectMetadata.nameSingular === objectMetadataName,
);
const objectMetadata =
this.internalContext.objectMetadataMap[objectMetadataName];
if (!objectMetadata) {
throw new Error(
`Object metadata for object "${objectMetadataName}" is missing ` +
`in workspace "${this.internalContext.workspaceId}" ` +
`with object metadata collection length: ${this.internalContext.objectMetadataCollection.length}`,
`with object metadata collection length: ${this.internalContext.objectMetadataMap.length}`,
);
}
@ -639,10 +639,12 @@ export class WorkspaceRepository<
}
private async getCompositeFieldMetadataCollection(
objectMetadata: ObjectMetadataEntity,
objectMetadata: ObjectMetadataMapItem,
) {
const compositeFieldMetadataCollection = objectMetadata.fields.filter(
(fieldMetadata) => isCompositeFieldMetadataType(fieldMetadata.type),
const compositeFieldMetadataCollection = Object.values(
objectMetadata.fields,
).filter((fieldMetadata) =>
isCompositeFieldMetadataType(fieldMetadata.type),
);
return compositeFieldMetadataCollection;
@ -723,7 +725,7 @@ export class WorkspaceRepository<
private async formatResult<T>(
data: T,
objectMetadata?: ObjectMetadataEntity,
objectMetadata?: ObjectMetadataMapItem,
): Promise<T> {
objectMetadata ??= await this.getObjectMetadataFromTarget();
@ -767,7 +769,7 @@ export class WorkspaceRepository<
);
const relationMetadataMap = new Map(
objectMetadata.fields
Object.values(objectMetadata.fields)
.filter(({ type }) => isRelationFieldMetadataType(type))
.map((fieldMetadata) => [
fieldMetadata.name,
@ -778,7 +780,7 @@ export class WorkspaceRepository<
relationType: computeRelationType(
fieldMetadata,
fieldMetadata.fromRelationMetadata ??
fieldMetadata.toRelationMetadata,
(fieldMetadata.toRelationMetadata as RelationMetadataEntity),
),
},
]),
@ -801,16 +803,14 @@ export class WorkspaceRepository<
if (relationMetadata) {
const toObjectMetadata =
this.internalContext.objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.toObjectMetadataId,
);
this.internalContext.objectMetadataMap[
relationMetadata.toObjectMetadataId
];
const fromObjectMetadata =
this.internalContext.objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.fromObjectMetadataId,
);
this.internalContext.objectMetadataMap[
relationMetadata.fromObjectMetadataId
];
if (!toObjectMetadata) {
throw new Error(

7
packages/twenty-server/src/engine/twenty-orm/utils/compute-relation-type.util.ts
View File

@ -1,15 +1,16 @@
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import {
RelationMetadataEntity,
RelationMetadataType,
} from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import {
deduceRelationDirection,
RelationDirection,
deduceRelationDirection,
} from 'src/engine/utils/deduce-relation-direction.util';
export const computeRelationType = (
fieldMetadata: FieldMetadataEntity,
fieldMetadata: FieldMetadataInterface,
relationMetadata: RelationMetadataEntity,
) => {
const relationDirection = deduceRelationDirection(

34
packages/twenty-server/src/engine/twenty-orm/utils/determine-relation-details.util.ts
View File

@ -1,8 +1,9 @@
import { RelationType } from 'typeorm/metadata/types/RelationTypes';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import { ObjectMetadataMap } from 'src/engine/metadata-modules/utils/generate-object-metadata-map.util';
import { computeRelationType } from 'src/engine/twenty-orm/utils/compute-relation-type.util';
interface RelationDetails {
@ -13,37 +14,28 @@ interface RelationDetails {
}
export async function determineRelationDetails(
fieldMetadata: FieldMetadataEntity,
fieldMetadata: FieldMetadataInterface,
relationMetadata: RelationMetadataEntity,
objectMetadataCollection: ObjectMetadataEntity[],
objectMetadataMap: ObjectMetadataMap,
): Promise<RelationDetails> {
const relationType = computeRelationType(fieldMetadata, relationMetadata);
let fromObjectMetadata: ObjectMetadataEntity | undefined =
fieldMetadata.object;
let toObjectMetadata: ObjectMetadataEntity | undefined =
objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.toObjectMetadataId,
);
const fromObjectMetadata = objectMetadataMap[fieldMetadata.objectMetadataId];
let toObjectMetadata = objectMetadataMap[relationMetadata.toObjectMetadataId];
// RelationMetadata always store the relation from the perspective of the `from` object, MANY_TO_ONE relations are not stored yet
if (relationType === 'many-to-one') {
fromObjectMetadata = fieldMetadata.object;
toObjectMetadata = objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.fromObjectMetadataId,
);
toObjectMetadata = objectMetadataMap[relationMetadata.fromObjectMetadataId];
}
if (!fromObjectMetadata || !toObjectMetadata) {
throw new Error('Object metadata not found');
}
const toFieldMetadata = toObjectMetadata.fields.find((field) =>
relationType === 'many-to-one'
? field.id === relationMetadata.fromFieldMetadataId
: field.id === relationMetadata.toFieldMetadataId,
const toFieldMetadata = Object.values(toObjectMetadata.fields).find(
(field) =>
relationType === 'many-to-one'
? field.id === relationMetadata.fromFieldMetadataId
: field.id === relationMetadata.toFieldMetadataId,
);
if (!toFieldMetadata) {