admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Improve performance on metadata computation (#12785)

Browse Source 
In this PR:

## Improve recompute metadata cache performance. We are aiming for
~100ms

Deleting relationMetadata table and FKs pointing on it
Fetching indexMetadata and indexFieldMetadata in a separate query as
typeorm is suboptimizing

## Remove caching lock

As recomputing the metadata cache is lighter, we try to stop preventing
multiple concurrent computations. This also simplifies interfaces

## Introduce self recovery mecanisms to recompute cache automatically if
corrupted

Aka getFreshObjectMetadataMaps

## custom object resolver performance improvement:  1sec to 200ms

Double check queries and indexes used while creating a custom object
Remove the queries to db to use the cached objectMetadataMap

## reduce objectMetadataMaps to 500kb
<img width="222" alt="image"
src="https://github.com/user-attachments/assets/2370dc80-49b6-4b63-8d5e-30c5ebdaa062"
/>

We used to stored 3 fieldMetadataMaps (byId, byName, byJoinColumnName).
While this is great for devXP, this is not great for performances.
Using the same mecanisme as for objectMetadataMap: we only keep byIdMap
and introduce two otherMaps to idByName, idByJoinColumnName to make the
bridge

## Add dataloader on IndexMetadata (aka indexMetadataList in the API)

## Improve field resolver performances too

## Deprecate ClientConfig
This commit is contained in:
Charles Bochet
2025-06-23 21:06:17 +02:00
committed by GitHub
parent 6aee42ab22
commit d5c974054d
145 changed files with 1485 additions and 2245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-create-many-resolver.service.ts
View File

@ -12,6 +12,10 @@ import { ObjectRecord } from 'src/engine/api/graphql/workspace-query-builder/int
import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface';
import { CreateManyResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { assertIsValidUuid } from 'src/engine/api/graphql/workspace-query-runner/utils/assert-is-valid-uuid.util';
import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type';
@ -19,6 +23,7 @@ import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-meta
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
@ -128,7 +133,7 @@ export class GraphqlQueryCreateManyResolverService extends GraphqlQueryBaseResol
fullPath: string;
column: string;
}[] {
return objectMetadataItemWithFieldMaps.fields
return Object.values(objectMetadataItemWithFieldMaps.fieldsById)
.filter((field) => field.isUnique || field.name === 'id')
.flatMap((field) => {
const compositeType = compositeTypeDefinitions.get(field.type);
@ -330,7 +335,10 @@ export class GraphqlQueryCreateManyResolverService extends GraphqlQueryBaseResol
records: structuredClone([record]),
updatedFields: Object.keys(formattedPartialRecordToUpdate),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem:
getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
}
}
@ -373,7 +381,9 @@ export class GraphqlQueryCreateManyResolverService extends GraphqlQueryBaseResol
this.apiEventEmitterService.emitCreateEvents({
records: structuredClone(formattedInsertedRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
}
@ -450,11 +460,19 @@ export class GraphqlQueryCreateManyResolverService extends GraphqlQueryBaseResol
) {
let recordWithoutCreatedByUpdate = record;
if (
'createdBy' in record &&
objectMetadataItemWithFieldMaps.fieldsByName['createdBy']?.isCustom ===
false
) {
const createdByFieldMetadataId =
objectMetadataItemWithFieldMaps.fieldIdByName['createdBy'];
const createdByFieldMetadata =
objectMetadataItemWithFieldMaps.fieldsById[createdByFieldMetadataId];
if (!isDefined(createdByFieldMetadata)) {
throw new GraphqlQueryRunnerException(
`Missing createdBy field metadata for object ${objectMetadataItemWithFieldMaps.nameSingular}`,
GraphqlQueryRunnerExceptionCode.MISSING_SYSTEM_FIELD,
);
}
if ('createdBy' in record && createdByFieldMetadata.isCustom === false) {
const { createdBy: _createdBy, ...recordWithoutCreatedBy } = record;
recordWithoutCreatedByUpdate = recordWithoutCreatedBy;

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-create-one-resolver.service.ts
View File

@ -14,6 +14,7 @@ import { CreateOneResolverArgs } from 'src/engine/api/graphql/workspace-resolver
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { assertIsValidUuid } from 'src/engine/api/graphql/workspace-query-runner/utils/assert-is-valid-uuid.util';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
@Injectable()
@ -56,7 +57,9 @@ export class GraphqlQueryCreateOneResolverService extends GraphqlQueryBaseResolv
this.apiEventEmitterService.emitCreateEvents({
records: structuredClone(upsertedRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-delete-many-resolver.service.ts
View File

@ -13,6 +13,7 @@ import { DeleteManyResolverArgs } from 'src/engine/api/graphql/workspace-resolve
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { assertIsValidUuid } from 'src/engine/api/graphql/workspace-query-runner/utils/assert-is-valid-uuid.util';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
@ -58,7 +59,9 @@ export class GraphqlQueryDeleteManyResolverService extends GraphqlQueryBaseResol
this.apiEventEmitterService.emitDeletedEvents({
records: structuredClone(formattedDeletedRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-delete-one-resolver.service.ts
View File

@ -18,6 +18,7 @@ import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/g
import { assertIsValidUuid } from 'src/engine/api/graphql/workspace-query-runner/utils/assert-is-valid-uuid.util';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
@Injectable()
export class GraphqlQueryDeleteOneResolverService extends GraphqlQueryBaseResolverService<
@ -60,7 +61,9 @@ export class GraphqlQueryDeleteOneResolverService extends GraphqlQueryBaseResolv
this.apiEventEmitterService.emitDeletedEvents({
records: structuredClone(formattedDeletedRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-destroy-many-resolver.service.ts
View File

@ -11,6 +11,7 @@ import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-qu
import { DestroyManyResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
@ -56,7 +57,9 @@ export class GraphqlQueryDestroyManyResolverService extends GraphqlQueryBaseReso
this.apiEventEmitterService.emitDestroyEvents({
records: structuredClone(deletedRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-destroy-one-resolver.service.ts
View File

@ -15,6 +15,7 @@ import {
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
@Injectable()
@ -56,7 +57,9 @@ export class GraphqlQueryDestroyOneResolverService extends GraphqlQueryBaseResol
this.apiEventEmitterService.emitDestroyEvents({
records: structuredClone(deletedRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-duplicates-resolver.service.ts
View File

@ -21,10 +21,10 @@ import {
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser';
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util';
import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';
@Injectable()
export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseResolverService<
@ -56,8 +56,7 @@ export class GraphqlQueryFindDuplicatesResolverService extends GraphqlQueryBaseR
}
const graphqlQueryParser = new GraphqlQueryParser(
objectMetadataItemWithFieldsMaps?.fieldsByName,
objectMetadataItemWithFieldsMaps?.fieldsByJoinColumnName,
objectMetadataItemWithFieldMaps,
objectMetadataMaps,
);

2
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-find-many-resolver.service.ts
View File

@ -80,7 +80,7 @@ export class GraphqlQueryFindManyResolverService extends GraphqlQueryBaseResolve
const cursorArgFilter = computeCursorArgFilter(
cursor,
orderByWithIdCondition,
objectMetadataItemWithFieldMaps.fieldsByName,
objectMetadataItemWithFieldMaps,
isForwardPagination,
);

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-restore-many-resolver.service.ts
View File

@ -15,6 +15,7 @@ import { assertIsValidUuid } from 'src/engine/api/graphql/workspace-query-runner
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
@Injectable()
export class GraphqlQueryRestoreManyResolverService extends GraphqlQueryBaseResolverService<
@ -58,7 +59,9 @@ export class GraphqlQueryRestoreManyResolverService extends GraphqlQueryBaseReso
this.apiEventEmitterService.emitRestoreEvents({
records: structuredClone(formattedRestoredRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-restore-one-resolver.service.ts
View File

@ -17,6 +17,7 @@ import {
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { assertIsValidUuid } from 'src/engine/api/graphql/workspace-query-runner/utils/assert-is-valid-uuid.util';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
@Injectable()
@ -60,7 +61,9 @@ export class GraphqlQueryRestoreOneResolverService extends GraphqlQueryBaseResol
this.apiEventEmitterService.emitRestoreEvents({
records: structuredClone(formattedRestoredRecords),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-update-many-resolver.service.ts
View File

@ -21,6 +21,7 @@ import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/obj
import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
import { computeTableName } from 'src/engine/utils/compute-table-name.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
@Injectable()
export class GraphqlQueryUpdateManyResolverService extends GraphqlQueryBaseResolverService<
@ -94,7 +95,9 @@ export class GraphqlQueryUpdateManyResolverService extends GraphqlQueryBaseResol
records: structuredClone(formattedUpdatedRecords),
updatedFields: Object.keys(executionArgs.args.data),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {

5
packages/twenty-server/src/engine/api/graphql/graphql-query-runner/resolvers/graphql-query-update-one-resolver.service.ts
View File

@ -18,6 +18,7 @@ import {
import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper';
import { assertIsValidUuid } from 'src/engine/api/graphql/workspace-query-runner/utils/assert-is-valid-uuid.util';
import { assertMutationNotOnRemoteObject } from 'src/engine/metadata-modules/object-metadata/utils/assert-mutation-not-on-remote-object.util';
import { getObjectMetadataFromObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/utils/get-object-metadata-from-object-metadata-Item-with-field-maps';
import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util';
@ -89,7 +90,9 @@ export class GraphqlQueryUpdateOneResolverService extends GraphqlQueryBaseResolv
records: structuredClone(formattedUpdatedRecords),
updatedFields: Object.keys(executionArgs.args.data),
authContext,
objectMetadataItem: objectMetadataItemWithFieldMaps,
objectMetadataItem: getObjectMetadataFromObjectMetadataItemWithFieldMaps(
objectMetadataItemWithFieldMaps,
),
});
if (executionArgs.graphqlQuerySelectedFieldsResult.relations) {