[POC] add graphql query runner (#6747)

## Context
The goal is to replace pg_graphql with our own ORM wrapper (TwentyORM).
This PR tries to add some parsing logic to convert graphql requests to
send to the ORM to replace pg_graphql implementation.

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Weiko
2024-08-27 17:06:39 +02:00
committed by GitHub
parent ef4f2e43b0
commit f6fd92adcb
51 changed files with 1397 additions and 249 deletions

View File

@ -0,0 +1 @@
export const CONNECTION_MAX_DEPTH = 5;

View File

@ -0,0 +1 @@
export const QUERY_MAX_RECORDS = 60;

View File

@ -0,0 +1,17 @@
import { CustomException } from 'src/utils/custom-exception';
export class GraphqlQueryRunnerException extends CustomException {
code: GraphqlQueryRunnerExceptionCode;
constructor(message: string, code: GraphqlQueryRunnerExceptionCode) {
super(message, code);
}
}
export enum GraphqlQueryRunnerExceptionCode {
MAX_DEPTH_REACHED = 'MAX_DEPTH_REACHED',
INVALID_CURSOR = 'INVALID_CURSOR',
INVALID_DIRECTION = 'INVALID_DIRECTION',
UNSUPPORTED_OPERATOR = 'UNSUPPORTED_OPERATOR',
ARGS_CONFLICT = 'ARGS_CONFLICT',
FIELD_NOT_FOUND = 'FIELD_NOT_FOUND',
}

View File

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { GraphqlQueryRunnerService } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-runner.service';
@Module({
providers: [GraphqlQueryRunnerService],
exports: [GraphqlQueryRunnerService],
})
export class GraphqlQueryRunnerModule {}

View File

@ -0,0 +1,130 @@
import { Injectable } from '@nestjs/common';
import graphqlFields from 'graphql-fields';
import { FindManyOptions, ObjectLiteral } from 'typeorm';
import {
Record as IRecord,
RecordFilter,
RecordOrderBy,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { IConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/connection.interface';
import { WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface';
import { FindManyResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { QUERY_MAX_RECORDS } from 'src/engine/api/graphql/graphql-query-runner/constants/query-max-records.constant';
import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query.parser';
import { applyRangeFilter } from 'src/engine/api/graphql/graphql-query-runner/utils/apply-range-filter.util';
import {
createConnection,
decodeCursor,
} from 'src/engine/api/graphql/graphql-query-runner/utils/connection.util';
import { convertObjectMetadataToMap } from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
import { LogExecutionTime } from 'src/engine/decorators/observability/log-execution-time.decorator';
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
@Injectable()
export class GraphqlQueryRunnerService {
constructor(
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
) {}
@LogExecutionTime()
async findManyWithTwentyOrm<
ObjectRecord extends IRecord = IRecord,
Filter extends RecordFilter = RecordFilter,
OrderBy extends RecordOrderBy = RecordOrderBy,
>(
args: FindManyResolverArgs<Filter, OrderBy>,
options: WorkspaceQueryRunnerOptions,
): Promise<IConnection<ObjectRecord>> {
const { authContext, objectMetadataItem, info, objectMetadataCollection } =
options;
const repository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
authContext.workspace.id,
objectMetadataItem.nameSingular,
);
const selectedFields = graphqlFields(info);
const objectMetadataMap = convertObjectMetadataToMap(
objectMetadataCollection,
);
const objectMetadata = objectMetadataMap[objectMetadataItem.nameSingular];
if (!objectMetadata) {
throw new Error(
`Object metadata for ${objectMetadataItem.nameSingular} not found`,
);
}
const fieldMetadataMap = objectMetadata.fields;
const graphqlQueryParser = new GraphqlQueryParser(
fieldMetadataMap,
objectMetadataMap,
);
const { select, relations } = graphqlQueryParser.parseSelectedFields(
objectMetadataItem,
selectedFields,
);
const order = args.orderBy
? graphqlQueryParser.parseOrder(args.orderBy)
: undefined;
const where = args.filter
? graphqlQueryParser.parseFilter(args.filter)
: {};
let cursor: Record<string, any> | undefined;
if (args.after) {
cursor = decodeCursor(args.after);
} else if (args.before) {
cursor = decodeCursor(args.before);
}
if (args.first && args.last) {
throw new GraphqlQueryRunnerException(
'Cannot provide both first and last',
GraphqlQueryRunnerExceptionCode.ARGS_CONFLICT,
);
}
const take = args.first ?? args.last ?? QUERY_MAX_RECORDS;
const findOptions: FindManyOptions<ObjectLiteral> = {
where,
order,
select,
relations,
take,
};
const totalCount = await repository.count({
where,
});
if (cursor) {
applyRangeFilter(where, order, cursor);
}
const objectRecords = await repository.find(findOptions);
return createConnection(
(objectRecords as ObjectRecord[]) ?? [],
take,
totalCount,
order,
);
}
}

View File

@ -0,0 +1,43 @@
import { FindOperator, Not } from 'typeorm';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { GraphqlQueryFilterFieldParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-filter/graphql-query-filter-field.parser';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
describe('GraphqlQueryFilterFieldParser', () => {
let parser: GraphqlQueryFilterFieldParser;
let mockFieldMetadataMap: Record<string, FieldMetadataInterface>;
beforeEach(() => {
mockFieldMetadataMap = {
simpleField: {
id: '1',
name: 'simpleField',
type: FieldMetadataType.TEXT,
label: 'Simple Field',
objectMetadataId: 'obj1',
},
};
parser = new GraphqlQueryFilterFieldParser(mockFieldMetadataMap);
});
it('should parse simple field correctly', () => {
const result = parser.parse('simpleField', 'value', false);
expect(result).toEqual({ simpleField: 'value' });
});
it('should negate simple field correctly', () => {
const result = parser.parse('simpleField', 'value', true);
expect(result).toEqual({ simpleField: Not('value') });
});
it('should parse object value using operator parser', () => {
const result = parser.parse('simpleField', { like: '%value%' }, false);
expect(result).toEqual({
simpleField: new FindOperator('like', '%%value%%'),
});
});
});

View File

@ -0,0 +1,130 @@
import {
FindOperator,
ILike,
In,
IsNull,
LessThan,
LessThanOrEqual,
Like,
MoreThan,
MoreThanOrEqual,
Not,
} from 'typeorm';
import { GraphqlQueryRunnerException } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { GraphqlQueryFilterOperatorParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-filter/graphql-query-filter-operator.parser';
describe('GraphqlQueryFilterOperatorParser', () => {
let parser: GraphqlQueryFilterOperatorParser;
beforeEach(() => {
parser = new GraphqlQueryFilterOperatorParser();
});
describe('parseOperator', () => {
it('should parse eq operator correctly', () => {
const result = parser.parseOperator({ eq: 'value' }, false);
expect(result).toBe('value');
});
it('should parse neq operator correctly', () => {
const result = parser.parseOperator({ neq: 'value' }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(Not('value'));
});
it('should parse gt operator correctly', () => {
const result = parser.parseOperator({ gt: 5 }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(MoreThan(5));
});
it('should parse gte operator correctly', () => {
const result = parser.parseOperator({ gte: 5 }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(MoreThanOrEqual(5));
});
it('should parse lt operator correctly', () => {
const result = parser.parseOperator({ lt: 5 }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(LessThan(5));
});
it('should parse lte operator correctly', () => {
const result = parser.parseOperator({ lte: 5 }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(LessThanOrEqual(5));
});
it('should parse in operator correctly', () => {
const result = parser.parseOperator({ in: [1, 2, 3] }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(In([1, 2, 3]));
});
it('should parse is operator with NULL correctly', () => {
const result = parser.parseOperator({ is: 'NULL' }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(IsNull());
});
it('should parse is operator with non-NULL value correctly', () => {
const result = parser.parseOperator({ is: 'NOT_NULL' }, false);
expect(result).toBe('NOT_NULL');
});
it('should parse like operator correctly', () => {
const result = parser.parseOperator({ like: 'test' }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(Like('%test%'));
});
it('should parse ilike operator correctly', () => {
const result = parser.parseOperator({ ilike: 'test' }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(ILike('%test%'));
});
it('should parse startsWith operator correctly', () => {
const result = parser.parseOperator({ startsWith: 'test' }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(ILike('test%'));
});
it('should parse endsWith operator correctly', () => {
const result = parser.parseOperator({ endsWith: 'test' }, false);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(ILike('%test'));
});
it('should negate the operator when isNegated is true', () => {
const result = parser.parseOperator({ eq: 'value' }, true);
expect(result).toBeInstanceOf(FindOperator);
expect(result).toEqual(Not('value'));
});
it('should throw an exception for unsupported operator', () => {
expect(() =>
parser.parseOperator({ unsupported: 'value' }, false),
).toThrow(GraphqlQueryRunnerException);
expect(() =>
parser.parseOperator({ unsupported: 'value' }, false),
).toThrow('Operator "unsupported" is not supported');
});
});
});

View File

@ -0,0 +1,107 @@
import { FindOptionsWhere, ObjectLiteral } from 'typeorm';
import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { FieldMetadataMap } from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
import { GraphqlQueryFilterFieldParser } from './graphql-query-filter-field.parser';
export class GraphqlQueryFilterConditionParser {
private fieldMetadataMap: FieldMetadataMap;
private fieldConditionParser: GraphqlQueryFilterFieldParser;
constructor(fieldMetadataMap: FieldMetadataMap) {
this.fieldMetadataMap = fieldMetadataMap;
this.fieldConditionParser = new GraphqlQueryFilterFieldParser(
this.fieldMetadataMap,
);
}
public parse(
conditions: RecordFilter,
isNegated = false,
): FindOptionsWhere<ObjectLiteral> | FindOptionsWhere<ObjectLiteral>[] {
if (Array.isArray(conditions)) {
return this.parseAndCondition(conditions, isNegated);
}
const result: FindOptionsWhere<ObjectLiteral> = {};
for (const [key, value] of Object.entries(conditions)) {
switch (key) {
case 'and':
return this.parseAndCondition(value, isNegated);
case 'or':
return this.parseOrCondition(value, isNegated);
case 'not':
return this.parse(value, !isNegated);
default:
Object.assign(
result,
this.fieldConditionParser.parse(key, value, isNegated),
);
}
}
return result;
}
private parseAndCondition(
conditions: RecordFilter[],
isNegated: boolean,
): FindOptionsWhere<ObjectLiteral>[] {
const parsedConditions = conditions.map((condition) =>
this.parse(condition, isNegated),
);
return this.combineConditions(parsedConditions, isNegated ? 'or' : 'and');
}
private parseOrCondition(
conditions: RecordFilter[],
isNegated: boolean,
): FindOptionsWhere<ObjectLiteral>[] {
const parsedConditions = conditions.map((condition) =>
this.parse(condition, isNegated),
);
return this.combineConditions(parsedConditions, isNegated ? 'and' : 'or');
}
private combineConditions(
conditions: (
| FindOptionsWhere<ObjectLiteral>
| FindOptionsWhere<ObjectLiteral>[]
)[],
combineType: 'and' | 'or',
): FindOptionsWhere<ObjectLiteral>[] {
if (combineType === 'and') {
let result: FindOptionsWhere<ObjectLiteral>[] = [{}];
for (const condition of conditions) {
if (Array.isArray(condition)) {
const newResult: FindOptionsWhere<ObjectLiteral>[] = [];
for (const existingCondition of result) {
for (const orCondition of condition) {
newResult.push({
...existingCondition,
...orCondition,
});
}
}
result = newResult;
} else {
result = result.map((existingCondition) => ({
...existingCondition,
...condition,
}));
}
}
return result;
}
return conditions.flat();
}
}

View File

@ -0,0 +1,102 @@
import { FindOptionsWhere, Not, ObjectLiteral } from 'typeorm';
import { RecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { FieldMetadataMap } from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { capitalize } from 'src/utils/capitalize';
import { isPlainObject } from 'src/utils/is-plain-object';
import { GraphqlQueryFilterConditionParser } from './graphql-query-filter-condition.parser';
import { GraphqlQueryFilterOperatorParser } from './graphql-query-filter-operator.parser';
export class GraphqlQueryFilterFieldParser {
private fieldMetadataMap: FieldMetadataMap;
private operatorParser: GraphqlQueryFilterOperatorParser;
constructor(fieldMetadataMap: FieldMetadataMap) {
this.fieldMetadataMap = fieldMetadataMap;
this.operatorParser = new GraphqlQueryFilterOperatorParser();
}
public parse(
key: string,
value: any,
isNegated: boolean,
): FindOptionsWhere<ObjectLiteral> {
const fieldMetadata = this.fieldMetadataMap[key];
if (!fieldMetadata) {
return {
[key]: (value: RecordFilter, isNegated: boolean) => {
const conditionParser = new GraphqlQueryFilterConditionParser(
this.fieldMetadataMap,
);
return conditionParser.parse(value, isNegated);
},
};
}
if (isCompositeFieldMetadataType(fieldMetadata.type)) {
return this.parseCompositeFieldForFilter(fieldMetadata, value, isNegated);
}
if (isPlainObject(value)) {
const parsedValue = this.operatorParser.parseOperator(value, isNegated);
return { [key]: parsedValue };
}
return { [key]: isNegated ? Not(value) : value };
}
private parseCompositeFieldForFilter(
fieldMetadata: FieldMetadataInterface,
fieldValue: any,
isNegated: boolean,
): FindOptionsWhere<ObjectLiteral> {
const compositeType = compositeTypeDefinitions.get(
fieldMetadata.type as CompositeFieldMetadataType,
);
if (!compositeType) {
throw new Error(
`Composite type definition not found for type: ${fieldMetadata.type}`,
);
}
return Object.entries(fieldValue).reduce(
(result, [subFieldKey, subFieldValue]) => {
const subFieldMetadata = compositeType.properties.find(
(property) => property.name === subFieldKey,
);
if (!subFieldMetadata) {
throw new Error(
`Sub field metadata not found for composite type: ${fieldMetadata.type}`,
);
}
const fullFieldName = `${fieldMetadata.name}${capitalize(subFieldKey)}`;
if (isPlainObject(subFieldValue)) {
result[fullFieldName] = this.operatorParser.parseOperator(
subFieldValue,
isNegated,
);
} else {
result[fullFieldName] = isNegated
? Not(subFieldValue)
: subFieldValue;
}
return result;
},
{} as FindOptionsWhere<ObjectLiteral>,
);
}
}

View File

@ -0,0 +1,56 @@
import {
FindOperator,
ILike,
In,
IsNull,
LessThan,
LessThanOrEqual,
Like,
MoreThan,
MoreThanOrEqual,
Not,
} from 'typeorm';
import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
export class GraphqlQueryFilterOperatorParser {
private operatorMap: { [key: string]: (value: any) => FindOperator<any> };
constructor() {
this.operatorMap = {
eq: (value: any) => value,
neq: (value: any) => Not(value),
gt: (value: any) => MoreThan(value),
gte: (value: any) => MoreThanOrEqual(value),
lt: (value: any) => LessThan(value),
lte: (value: any) => LessThanOrEqual(value),
in: (value: any) => In(value),
is: (value: any) => (value === 'NULL' ? IsNull() : value),
like: (value: string) => Like(`%${value}%`),
ilike: (value: string) => ILike(`%${value}%`),
startsWith: (value: string) => ILike(`${value}%`),
endsWith: (value: string) => ILike(`%${value}`),
};
}
public parseOperator(
operatorObj: Record<string, any>,
isNegated: boolean,
): FindOperator<any> {
const [[operator, value]] = Object.entries(operatorObj);
if (operator in this.operatorMap) {
const operatorFunction = this.operatorMap[operator];
return isNegated ? Not(operatorFunction(value)) : operatorFunction(value);
}
throw new GraphqlQueryRunnerException(
`Operator "${operator}" is not supported`,
GraphqlQueryRunnerExceptionCode.UNSUPPORTED_OPERATOR,
);
}
}

View File

@ -0,0 +1,50 @@
import { OrderByDirection } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { GraphqlQueryOrderFieldParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-order/graphql-query-order.parser';
import { FieldMetadataMap } from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
describe('GraphqlQueryOrderFieldParser', () => {
let parser: GraphqlQueryOrderFieldParser;
const fieldMetadataMap: FieldMetadataMap = {};
beforeEach(() => {
fieldMetadataMap['name'] = {
id: 'name-id',
name: 'name',
type: FieldMetadataType.TEXT,
label: 'Name',
objectMetadataId: 'object-id',
};
fieldMetadataMap['age'] = {
id: 'age-id',
name: 'age',
type: FieldMetadataType.NUMBER,
label: 'Age',
objectMetadataId: 'object-id',
};
fieldMetadataMap['address'] = {
id: 'address-id',
name: 'address',
type: FieldMetadataType.ADDRESS,
label: 'Address',
objectMetadataId: 'object-id',
};
parser = new GraphqlQueryOrderFieldParser(fieldMetadataMap);
});
describe('parse', () => {
it('should parse simple order by fields', () => {
const orderBy = [
{ name: OrderByDirection.AscNullsFirst },
{ age: OrderByDirection.DescNullsLast },
];
const result = parser.parse(orderBy);
expect(result).toEqual({
name: { direction: 'ASC', nulls: 'FIRST' },
age: { direction: 'DESC', nulls: 'LAST' },
});
});
});
});

View File

@ -0,0 +1,122 @@
import { FindOptionsOrderValue } from 'typeorm';
import {
OrderByDirection,
RecordOrderBy,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { FieldMetadataMap } from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { capitalize } from 'src/utils/capitalize';
export class GraphqlQueryOrderFieldParser {
private fieldMetadataMap: FieldMetadataMap;
constructor(fieldMetadataMap: FieldMetadataMap) {
this.fieldMetadataMap = fieldMetadataMap;
}
parse(orderBy: RecordOrderBy): Record<string, FindOptionsOrderValue> {
return orderBy.reduce(
(acc, item) => {
Object.entries(item).forEach(([key, value]) => {
const fieldMetadata = this.fieldMetadataMap[key];
if (!fieldMetadata || value === undefined) {
throw new GraphqlQueryRunnerException(
`Field "${key}" does not exist or is not sortable`,
GraphqlQueryRunnerExceptionCode.FIELD_NOT_FOUND,
);
}
if (isCompositeFieldMetadataType(fieldMetadata.type)) {
const compositeOrder = this.parseCompositeFieldForOrder(
fieldMetadata,
value,
);
Object.assign(acc, compositeOrder);
} else {
acc[key] = this.convertOrderByToFindOptionsOrder(value);
}
});
return acc;
},
{} as Record<string, FindOptionsOrderValue>,
);
}
private parseCompositeFieldForOrder(
fieldMetadata: FieldMetadataInterface,
value: any,
): Record<string, FindOptionsOrderValue> {
const compositeType = compositeTypeDefinitions.get(
fieldMetadata.type as CompositeFieldMetadataType,
);
if (!compositeType) {
throw new Error(
`Composite type definition not found for type: ${fieldMetadata.type}`,
);
}
return Object.entries(value).reduce(
(acc, [subFieldKey, subFieldValue]) => {
const subFieldMetadata = compositeType.properties.find(
(property) => property.name === subFieldKey,
);
if (!subFieldMetadata) {
throw new Error(
`Sub field metadata not found for composite type: ${fieldMetadata.type}`,
);
}
const fullFieldName = `${fieldMetadata.name}${capitalize(subFieldKey)}`;
if (!this.isOrderByDirection(subFieldValue)) {
throw new Error(
`Sub field order by value must be of type OrderByDirection, but got: ${subFieldValue}`,
);
}
acc[fullFieldName] =
this.convertOrderByToFindOptionsOrder(subFieldValue);
return acc;
},
{} as Record<string, FindOptionsOrderValue>,
);
}
private convertOrderByToFindOptionsOrder(
direction: OrderByDirection,
): FindOptionsOrderValue {
switch (direction) {
case OrderByDirection.AscNullsFirst:
return { direction: 'ASC', nulls: 'FIRST' };
case OrderByDirection.AscNullsLast:
return { direction: 'ASC', nulls: 'LAST' };
case OrderByDirection.DescNullsFirst:
return { direction: 'DESC', nulls: 'FIRST' };
case OrderByDirection.DescNullsLast:
return { direction: 'DESC', nulls: 'LAST' };
default:
throw new GraphqlQueryRunnerException(
`Invalid direction: ${direction}`,
GraphqlQueryRunnerExceptionCode.INVALID_DIRECTION,
);
}
}
private isOrderByDirection(value: unknown): value is OrderByDirection {
return Object.values(OrderByDirection).includes(value as OrderByDirection);
}
}

View File

@ -0,0 +1,70 @@
import {
FindOptionsOrderValue,
FindOptionsWhere,
ObjectLiteral,
} from 'typeorm';
import {
RecordFilter,
RecordOrderBy,
} from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { GraphqlQueryFilterConditionParser as GraphqlQueryFilterParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-filter/graphql-query-filter-condition.parser';
import { GraphqlQueryOrderFieldParser as GraphqlQueryOrderParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-query-order/graphql-query-order.parser';
import { GraphQLSelectedFieldsParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-selected-fields/graphql-selected-fields.parser';
import {
FieldMetadataMap,
ObjectMetadataMap,
} from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
export class GraphqlQueryParser {
private fieldMetadataMap: FieldMetadataMap;
private objectMetadataMap: ObjectMetadataMap;
constructor(
fieldMetadataMap: FieldMetadataMap,
objectMetadataMap: ObjectMetadataMap,
) {
this.objectMetadataMap = objectMetadataMap;
this.fieldMetadataMap = fieldMetadataMap;
}
parseFilter(
recordFilter: RecordFilter,
): FindOptionsWhere<ObjectLiteral> | FindOptionsWhere<ObjectLiteral>[] {
const graphqlQueryFilterParser = new GraphqlQueryFilterParser(
this.fieldMetadataMap,
);
return graphqlQueryFilterParser.parse(recordFilter);
}
parseOrder(orderBy: RecordOrderBy): Record<string, FindOptionsOrderValue> {
const graphqlQueryOrderParser = new GraphqlQueryOrderParser(
this.fieldMetadataMap,
);
return graphqlQueryOrderParser.parse(orderBy);
}
parseSelectedFields(
parentObjectMetadata: ObjectMetadataInterface,
graphqlSelectedFields: Partial<Record<string, any>>,
): { select: Record<string, any>; relations: Record<string, any> } {
const parentFields =
this.objectMetadataMap[parentObjectMetadata.nameSingular]?.fields;
if (!parentFields) {
throw new Error(
`Could not find object metadata for ${parentObjectMetadata.nameSingular}`,
);
}
const selectedFieldsParser = new GraphQLSelectedFieldsParser(
this.objectMetadataMap,
);
return selectedFieldsParser.parse(graphqlSelectedFields, parentFields);
}
}

View File

@ -0,0 +1,79 @@
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { GraphQLSelectedFieldsParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-selected-fields/graphql-selected-fields.parser';
import {
ObjectMetadataMap,
ObjectMetadataMapItem,
} from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity';
import {
deduceRelationDirection,
RelationDirection,
} from 'src/engine/utils/deduce-relation-direction.util';
export class GraphqlSelectedFieldsRelationParser {
private objectMetadataMap: ObjectMetadataMap;
constructor(objectMetadataMap: ObjectMetadataMap) {
this.objectMetadataMap = objectMetadataMap;
}
parseRelationField(
fieldMetadata: FieldMetadataInterface,
fieldKey: string,
fieldValue: any,
result: { select: Record<string, any>; relations: Record<string, any> },
): void {
result.relations[fieldKey] = true;
if (!fieldValue || typeof fieldValue !== 'object') {
return;
}
const relationMetadata =
fieldMetadata.fromRelationMetadata ?? fieldMetadata.toRelationMetadata;
if (!relationMetadata) {
throw new Error(
`Relation metadata not found for field ${fieldMetadata.name}`,
);
}
const relationDirection = deduceRelationDirection(
fieldMetadata,
relationMetadata,
);
const referencedObjectMetadata = this.getReferencedObjectMetadata(
relationMetadata,
relationDirection,
);
const relationFields = referencedObjectMetadata.fields;
const fieldParser = new GraphQLSelectedFieldsParser(this.objectMetadataMap);
const subResult = fieldParser.parse(fieldValue, relationFields);
result.select[fieldKey] = {
id: true,
...subResult.select,
};
result.relations[fieldKey] = subResult.relations;
}
private getReferencedObjectMetadata(
relationMetadata: RelationMetadataEntity,
relationDirection: RelationDirection,
): ObjectMetadataMapItem {
const referencedObjectMetadata =
relationDirection === RelationDirection.TO
? this.objectMetadataMap[relationMetadata.fromObjectMetadataId]
: this.objectMetadataMap[relationMetadata.toObjectMetadataId];
if (!referencedObjectMetadata) {
throw new Error(
`Referenced object metadata not found for relation ${relationMetadata.id}`,
);
}
return referencedObjectMetadata;
}
}

View File

@ -0,0 +1,128 @@
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { GraphqlSelectedFieldsRelationParser } from 'src/engine/api/graphql/graphql-query-runner/parsers/graphql-selected-fields/graphql-selected-fields-relation.parser';
import { ObjectMetadataMap } from 'src/engine/api/graphql/graphql-query-runner/utils/convert-object-metadata-to-map.util';
import { compositeTypeDefinitions } from 'src/engine/metadata-modules/field-metadata/composite-types';
import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
import { CompositeFieldMetadataType } from 'src/engine/metadata-modules/workspace-migration/factories/composite-column-action.factory';
import { isRelationFieldMetadataType } from 'src/engine/utils/is-relation-field-metadata-type.util';
import { capitalize } from 'src/utils/capitalize';
import { isPlainObject } from 'src/utils/is-plain-object';
export class GraphQLSelectedFieldsParser {
private graphqlSelectedFieldsRelationParser: GraphqlSelectedFieldsRelationParser;
constructor(objectMetadataMap: ObjectMetadataMap) {
this.graphqlSelectedFieldsRelationParser =
new GraphqlSelectedFieldsRelationParser(objectMetadataMap);
}
parse(
graphqlSelectedFields: Partial<Record<string, any>>,
fieldMetadataMap: Record<string, FieldMetadataInterface>,
): { select: Record<string, any>; relations: Record<string, any> } {
const result: {
select: Record<string, any>;
relations: Record<string, any>;
} = {
select: {},
relations: {},
};
for (const [fieldKey, fieldValue] of Object.entries(
graphqlSelectedFields,
)) {
if (this.shouldNotParseField(fieldKey)) {
continue;
}
if (this.isConnectionField(fieldKey, fieldValue)) {
const subResult = this.parse(fieldValue, fieldMetadataMap);
Object.assign(result.select, subResult.select);
Object.assign(result.relations, subResult.relations);
continue;
}
const fieldMetadata = fieldMetadataMap[fieldKey];
if (!fieldMetadata) {
throw new GraphqlQueryRunnerException(
`Field "${fieldKey}" does not exist or is not selectable`,
GraphqlQueryRunnerExceptionCode.FIELD_NOT_FOUND,
);
}
if (isRelationFieldMetadataType(fieldMetadata.type)) {
this.graphqlSelectedFieldsRelationParser.parseRelationField(
fieldMetadata,
fieldKey,
fieldValue,
result,
);
} else if (isCompositeFieldMetadataType(fieldMetadata.type)) {
const compositeResult = this.parseCompositeField(
fieldMetadata,
fieldValue,
);
Object.assign(result.select, compositeResult);
} else {
result.select[fieldKey] = true;
}
}
return result;
}
private isConnectionField(fieldKey: string, fieldValue: any): boolean {
return ['edges', 'node'].includes(fieldKey) && isPlainObject(fieldValue);
}
private shouldNotParseField(fieldKey: string): boolean {
return ['__typename', 'totalCount', 'pageInfo', 'cursor'].includes(
fieldKey,
);
}
private parseCompositeField(
fieldMetadata: FieldMetadataInterface,
fieldValue: any,
): Record<string, any> {
const compositeType = compositeTypeDefinitions.get(
fieldMetadata.type as CompositeFieldMetadataType,
);
if (!compositeType) {
throw new Error(
`Composite type definition not found for type: ${fieldMetadata.type}`,
);
}
return Object.keys(fieldValue)
.filter((subFieldKey) => subFieldKey !== '__typename')
.reduce(
(acc, subFieldKey) => {
const subFieldMetadata = compositeType.properties.find(
(property) => property.name === subFieldKey,
);
if (!subFieldMetadata) {
throw new Error(
`Sub field metadata not found for composite type: ${fieldMetadata.type}`,
);
}
const fullFieldName = `${fieldMetadata.name}${capitalize(subFieldKey)}`;
acc[fullFieldName] = true;
return acc;
},
{} as Record<string, any>,
);
}
}

View File

@ -0,0 +1,29 @@
import {
FindOptionsOrderValue,
FindOptionsWhere,
LessThan,
MoreThan,
ObjectLiteral,
} from 'typeorm';
export const applyRangeFilter = (
where: FindOptionsWhere<ObjectLiteral>,
order: Record<string, FindOptionsOrderValue> | undefined,
cursor: Record<string, any>,
): FindOptionsWhere<ObjectLiteral> => {
if (!order) return where;
const orderEntries = Object.entries(order);
orderEntries.forEach(([column, order], index) => {
if (typeof order !== 'object' || !('direction' in order)) {
return;
}
where[column] =
order.direction === 'ASC'
? MoreThan(cursor[index])
: LessThan(cursor[index]);
});
return where;
};

View File

@ -0,0 +1,114 @@
import { FindOptionsOrderValue } from 'typeorm';
import { Record as IRecord } from 'src/engine/api/graphql/workspace-query-builder/interfaces/record.interface';
import { IConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/connection.interface';
import { CONNECTION_MAX_DEPTH } from 'src/engine/api/graphql/graphql-query-runner/constants/connection-max-depth.constant';
import {
GraphqlQueryRunnerException,
GraphqlQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
import { isPlainObject } from 'src/utils/is-plain-object';
export const createConnection = <ObjectRecord extends IRecord = IRecord>(
objectRecords: ObjectRecord[],
take: number,
totalCount: number,
order: Record<string, FindOptionsOrderValue> | undefined,
depth = 0,
): IConnection<ObjectRecord> => {
const edges = (objectRecords ?? []).map((objectRecord) => ({
node: processNestedConnections(
objectRecord,
take,
totalCount,
order,
depth,
),
cursor: encodeCursor(objectRecord, order),
}));
return {
edges,
pageInfo: {
hasNextPage: objectRecords.length === take && totalCount > take,
hasPreviousPage: false,
startCursor: edges[0]?.cursor,
endCursor: edges[edges.length - 1]?.cursor,
},
totalCount: totalCount,
};
};
const processNestedConnections = <T extends Record<string, any>>(
objectRecord: T,
take: number,
totalCount: number,
order: Record<string, FindOptionsOrderValue> | undefined,
depth = 0,
): T => {
if (depth >= CONNECTION_MAX_DEPTH) {
throw new GraphqlQueryRunnerException(
`Maximum depth of ${CONNECTION_MAX_DEPTH} reached`,
GraphqlQueryRunnerExceptionCode.MAX_DEPTH_REACHED,
);
}
const processedObjectRecords: Record<string, any> = { ...objectRecord };
for (const [key, value] of Object.entries(objectRecord)) {
if (Array.isArray(value)) {
if (value.length > 0 && typeof value[0] !== 'object') {
processedObjectRecords[key] = value;
} else {
processedObjectRecords[key] = createConnection(
value,
take,
value.length,
order,
depth + 1,
);
}
} else if (value instanceof Date) {
processedObjectRecords[key] = value.toISOString();
} else if (isPlainObject(value)) {
processedObjectRecords[key] = processNestedConnections(
value,
take,
totalCount,
order,
depth + 1,
);
} else {
processedObjectRecords[key] = value;
}
}
return processedObjectRecords as T;
};
export const decodeCursor = (cursor: string): Record<string, any> => {
try {
return JSON.parse(Buffer.from(cursor, 'base64').toString());
} catch (err) {
throw new GraphqlQueryRunnerException(
`Invalid cursor: ${cursor}`,
GraphqlQueryRunnerExceptionCode.INVALID_CURSOR,
);
}
};
export const encodeCursor = <ObjectRecord extends IRecord = IRecord>(
objectRecord: ObjectRecord,
order: Record<string, FindOptionsOrderValue> | undefined,
): string => {
const cursor = {};
Object.keys(order ?? []).forEach((key) => {
cursor[key] = objectRecord[key];
});
cursor['id'] = objectRecord.id;
return Buffer.from(JSON.stringify(Object.values(cursor))).toString('base64');
};

View File

@ -0,0 +1,35 @@
import { FieldMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/field-metadata.interface';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
export type FieldMetadataMap = Record<string, FieldMetadataInterface>;
export type ObjectMetadataMapItem = Omit<ObjectMetadataInterface, 'fields'> & {
fields: FieldMetadataMap;
};
export type ObjectMetadataMap = Record<string, ObjectMetadataMapItem>;
export const convertObjectMetadataToMap = (
objectMetadataCollection: ObjectMetadataInterface[],
): ObjectMetadataMap => {
const objectMetadataMap: ObjectMetadataMap = {};
for (const objectMetadata of objectMetadataCollection) {
const fieldsMap: FieldMetadataMap = {};
for (const fieldMetadata of objectMetadata.fields) {
fieldsMap[fieldMetadata.name] = fieldMetadata;
}
const processedObjectMetadata: ObjectMetadataMapItem = {
...objectMetadata,
fields: fieldsMap,
};
objectMetadataMap[objectMetadata.id] = processedObjectMetadata;
objectMetadataMap[objectMetadata.nameSingular] = processedObjectMetadata;
objectMetadataMap[objectMetadata.namePlural] = processedObjectMetadata;
}
return objectMetadataMap;
};

View File

@ -1,5 +1,7 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { GraphqlQueryRunnerModule } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module';
import { WorkspaceQueryBuilderModule } from 'src/engine/api/graphql/workspace-query-builder/workspace-query-builder.module';
import { RecordPositionBackfillCommand } from 'src/engine/api/graphql/workspace-query-runner/commands/0-20-record-position-backfill.command';
import { workspaceQueryRunnerFactories } from 'src/engine/api/graphql/workspace-query-runner/factories';
@ -8,6 +10,8 @@ import { WorkspaceQueryHookModule } from 'src/engine/api/graphql/workspace-query
import { AnalyticsModule } from 'src/engine/core-modules/analytics/analytics.module';
import { AuthModule } from 'src/engine/core-modules/auth/auth.module';
import { DuplicateModule } from 'src/engine/core-modules/duplicate/duplicate.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module';
import { FileModule } from 'src/engine/core-modules/file/file.module';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@ -24,9 +28,12 @@ import { EntityEventsToDbListener } from './listeners/entity-events-to-db.listen
WorkspaceDataSourceModule,
WorkspaceQueryHookModule,
ObjectMetadataRepositoryModule.forFeature([WorkspaceMemberWorkspaceEntity]),
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
AnalyticsModule,
DuplicateModule,
FileModule,
GraphqlQueryRunnerModule,
FeatureFlagModule,
],
providers: [
WorkspaceQueryRunnerService,

View File

@ -25,6 +25,7 @@ import {
} from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { GraphqlQueryRunnerService } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-runner.service';
import { WorkspaceQueryBuilderFactory } from 'src/engine/api/graphql/workspace-query-builder/workspace-query-builder.factory';
import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory';
import { QueryRunnerArgsFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory';
@ -42,6 +43,8 @@ import {
WorkspaceQueryRunnerExceptionCode,
} from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.exception';
import { DuplicateService } from 'src/engine/core-modules/duplicate/duplicate.service';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/types/object-record-delete.event';
@ -63,8 +66,8 @@ import {
} from './interfaces/pg-graphql.interface';
import { WorkspaceQueryRunnerOptions } from './interfaces/query-runner-option.interface';
import {
PgGraphQLConfig,
computePgGraphQLError,
PgGraphQLConfig,
} from './utils/compute-pg-graphql-error.util';
@Injectable()
@ -83,6 +86,8 @@ export class WorkspaceQueryRunnerService {
private readonly workspaceQueryHookService: WorkspaceQueryHookService,
private readonly environmentService: EnvironmentService,
private readonly duplicateService: DuplicateService,
private readonly featureFlagService: FeatureFlagService,
private readonly graphqlQueryRunnerService: GraphqlQueryRunnerService,
) {}
async findMany<
@ -96,6 +101,12 @@ export class WorkspaceQueryRunnerService {
const { authContext, objectMetadataItem } = options;
const start = performance.now();
const isQueryRunnerTwentyORMEnabled =
await this.featureFlagService.isFeatureEnabled(
FeatureFlagKey.IsQueryRunnerTwentyORMEnabled,
authContext.workspace.id,
);
const hookedArgs =
await this.workspaceQueryHookService.executePreQueryHooks(
authContext,
@ -110,6 +121,13 @@ export class WorkspaceQueryRunnerService {
ResolverArgsType.FindMany,
)) as FindManyResolverArgs<Filter, OrderBy>;
if (isQueryRunnerTwentyORMEnabled) {
return this.graphqlQueryRunnerService.findManyWithTwentyOrm(
computedArgs,
options,
);
}
const query = await this.workspaceQueryBuilderFactory.findMany(
computedArgs,
{
@ -119,6 +137,7 @@ export class WorkspaceQueryRunnerService {
);
const result = await this.execute(query, authContext.workspace.id);
const end = performance.now();
this.logger.log(

View File

@ -13,7 +13,6 @@ import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
@Injectable()
export class WorkspaceSchemaFactory {
constructor(