admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Sync remote object (#4713)

Browse Source 
* Sync objects

* Generate data for isRemote

* Add cache version update

* Add label identifier + fix field metadata input

---------

Co-authored-by: Thomas Trompette <thomast@twenty.com>
This commit is contained in:
Thomas Trompette
2024-03-29 18:23:58 +01:00
committed by GitHub
parent 7f3623239a
commit 1d351a29b8
22 changed files with 777 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/twenty-front/src/generated-metadata/gql.ts
View File

@ -20,7 +20,7 @@ const documents = {
"\n mutation UpdateOneObjectMetadataItem(\n $idToUpdate: ID!\n $updatePayload: UpdateObjectInput!\n ) {\n updateOneObject(input: { id: $idToUpdate, update: $updatePayload }) {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n }\n }\n": types.UpdateOneObjectMetadataItemDocument, "\n mutation UpdateOneObjectMetadataItem(\n $idToUpdate: ID!\n $updatePayload: UpdateObjectInput!\n ) {\n updateOneObject(input: { id: $idToUpdate, update: $updatePayload }) {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n }\n }\n": types.UpdateOneObjectMetadataItemDocument,
"\n mutation DeleteOneObjectMetadataItem($idToDelete: ID!) {\n deleteOneObject(input: { id: $idToDelete }) {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n }\n }\n": types.DeleteOneObjectMetadataItemDocument, "\n mutation DeleteOneObjectMetadataItem($idToDelete: ID!) {\n deleteOneObject(input: { id: $idToDelete }) {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n }\n }\n": types.DeleteOneObjectMetadataItemDocument,
"\n mutation DeleteOneFieldMetadataItem($idToDelete: ID!) {\n deleteOneField(input: { id: $idToDelete }) {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isNullable\n createdAt\n updatedAt\n }\n }\n": types.DeleteOneFieldMetadataItemDocument, "\n mutation DeleteOneFieldMetadataItem($idToDelete: ID!) {\n deleteOneField(input: { id: $idToDelete }) {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isNullable\n createdAt\n updatedAt\n }\n }\n": types.DeleteOneFieldMetadataItemDocument,
"\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n": types.ObjectMetadataItemsDocument, "\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isRemote\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n": types.ObjectMetadataItemsDocument,
}; };
/** /**
@ -68,7 +68,7 @@ export function graphql(source: "\n mutation DeleteOneFieldMetadataItem($idToDe
/** /**
* The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/ */
export function graphql(source: "\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"): (typeof documents)["\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"]; export function graphql(source: "\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isRemote\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"): (typeof documents)["\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isRemote\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"];
export function graphql(source: string) { export function graphql(source: string) {
return (documents as any)[source] ?? {}; return (documents as any)[source] ?? {};

21
packages/twenty-front/src/generated-metadata/graphql.ts
View File

File diff suppressed because one or more lines are too long

1
packages/twenty-front/src/modules/object-metadata/graphql/queries.ts
View File

@ -17,6 +17,7 @@ export const FIND_MANY_OBJECT_METADATA_ITEMS = gql`
description description
icon icon
isCustom isCustom
isRemote
isActive isActive
isSystem isSystem
createdAt createdAt

1
packages/twenty-front/src/modules/object-metadata/hooks/__mocks__/useFindManyObjectMetadataItems.ts
View File

@ -17,6 +17,7 @@ export const query = gql`
description description
icon icon
isCustom isCustom
isRemote
isActive isActive
isSystem isSystem
createdAt createdAt

7
packages/twenty-front/src/modules/object-metadata/utils/getObjectMetadataItemsMock.ts
View File

@ -13,6 +13,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A webhook', description: 'A webhook',
icon: 'IconRobot', icon: 'IconRobot',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: true, isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z', createdAt: '2023-11-30T11:13:15.206Z',
@ -30,6 +31,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'An api key', description: 'An api key',
icon: 'IconRobot', icon: 'IconRobot',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: true, isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z', createdAt: '2023-11-30T11:13:15.206Z',
@ -150,6 +152,7 @@ export const getObjectMetadataItemsMock = () => {
description: '(System) View Sorts', description: '(System) View Sorts',
icon: 'IconArrowsSort', icon: 'IconArrowsSort',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: true, isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z', createdAt: '2023-11-30T11:13:15.206Z',
@ -282,6 +285,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A calendar event', description: 'A calendar event',
icon: 'IconCalendarEvent', icon: 'IconCalendarEvent',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: true, isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z', createdAt: '2023-11-30T11:13:15.206Z',
@ -299,6 +303,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'An opportunity', description: 'An opportunity',
icon: 'IconTargetArrow', icon: 'IconTargetArrow',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: false, isSystem: false,
createdAt: '2023-11-30T11:13:15.206Z', createdAt: '2023-11-30T11:13:15.206Z',
@ -617,6 +622,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A person', description: 'A person',
icon: 'IconUser', icon: 'IconUser',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: false, isSystem: false,
createdAt: '2023-11-30T11:13:15.206Z', createdAt: '2023-11-30T11:13:15.206Z',
@ -1013,6 +1019,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A workspace member', description: 'A workspace member',
icon: 'IconUserCircle', icon: 'IconUserCircle',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: true, isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z', createdAt: '2023-11-30T11:13:15.206Z',

1
packages/twenty-front/src/modules/object-metadata/validation-schemas/objectMetadataItemSchema.ts
View File

@ -15,6 +15,7 @@ export const objectMetadataItemSchema = z.object({
imageIdentifierFieldMetadataId: z.string().uuid().nullable(), imageIdentifierFieldMetadataId: z.string().uuid().nullable(),
isActive: z.boolean(), isActive: z.boolean(),
isCustom: z.boolean(), isCustom: z.boolean(),
isRemote: z.boolean(),
isSystem: z.boolean(), isSystem: z.boolean(),
labelIdentifierFieldMetadataId: z.string().uuid().nullable(), labelIdentifierFieldMetadataId: z.string().uuid().nullable(),
labelPlural: z.string().trim().min(1), labelPlural: z.string().trim().min(1),

4
packages/twenty-front/src/modules/object-record/record-index/hooks/useLoadRecordIndexTable.ts
View File

@ -29,6 +29,10 @@ export const useFindManyParams = (
objectMetadataItem?.fields ?? [], objectMetadataItem?.fields ?? [],
); );
if (objectMetadataItem?.isRemote) {
return { objectNameSingular, filter };
}
const orderBy = turnSortsIntoOrderBy( const orderBy = turnSortsIntoOrderBy(
tableSorts, tableSorts,
objectMetadataItem?.fields ?? [], objectMetadataItem?.fields ?? [],

2
packages/twenty-front/src/testing/mock-data/metadata.ts
View File

@ -18,6 +18,7 @@ export const mockedPeopleMetadata = {
description: 'A person', description: 'A person',
icon: 'IconUser', icon: 'IconUser',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: false, isSystem: false,
createdAt: '2023-12-15T15:29:39.070Z', createdAt: '2023-12-15T15:29:39.070Z',
@ -594,6 +595,7 @@ export const mockedCompaniesMetadata = {
description: 'A company', description: 'A company',
icon: 'IconBuildingSkyscraper', icon: 'IconBuildingSkyscraper',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: false, isSystem: false,
createdAt: '2023-12-15T15:29:39.070Z', createdAt: '2023-12-15T15:29:39.070Z',

1
packages/twenty-front/src/testing/mock-data/objectMetadataItems.ts
View File

@ -11,6 +11,7 @@ export const mockObjectMetadataItem: ObjectMetadataItem = {
description: 'A company', description: 'A company',
icon: 'IconBuildingSkyscraper', icon: 'IconBuildingSkyscraper',
isCustom: false, isCustom: false,
isRemote: false,
isActive: true, isActive: true,
isSystem: false, isSystem: false,
createdAt: '2023-12-19T12:15:28.459Z', createdAt: '2023-12-19T12:15:28.459Z',

6
packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/create-field.input.ts
View File

@ -1,6 +1,6 @@
import { Field, InputType, OmitType } from '@nestjs/graphql'; import { Field, InputType, OmitType } from '@nestjs/graphql';
import { IsUUID, ValidateNested } from 'class-validator'; import { IsOptional, IsUUID, ValidateNested } from 'class-validator';
import { Type } from 'class-transformer'; import { Type } from 'class-transformer';
import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto'; import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
@ -14,6 +14,10 @@ export class CreateFieldInput extends OmitType(
@IsUUID() @IsUUID()
@Field() @Field()
objectMetadataId: string; objectMetadataId: string;
@Field(() => Boolean, { nullable: true })
@IsOptional()
isRemoteCreation?: boolean;
} }
@InputType() @InputType()

39
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
View File

@ -122,12 +122,13 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
...fieldMetadataInput, ...fieldMetadataInput,
targetColumnMap: generateTargetColumnMap( targetColumnMap: generateTargetColumnMap(
fieldMetadataInput.type, fieldMetadataInput.type,
true, !fieldMetadataInput.isRemoteCreation,
fieldMetadataInput.name, fieldMetadataInput.name,
), ),
isNullable: generateNullable( isNullable: generateNullable(
fieldMetadataInput.type, fieldMetadataInput.type,
fieldMetadataInput.isNullable, fieldMetadataInput.isNullable,
fieldMetadataInput.isRemoteCreation,
), ),
defaultValue: defaultValue:
fieldMetadataInput.defaultValue ?? fieldMetadataInput.defaultValue ??
@ -142,24 +143,26 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
isCustom: true, isCustom: true,
}); });
await this.workspaceMigrationService.createCustomMigration( if (!fieldMetadataInput.isRemoteCreation) {
generateMigrationName(`create-${createdFieldMetadata.name}`), await this.workspaceMigrationService.createCustomMigration(
fieldMetadataInput.workspaceId, generateMigrationName(`create-${createdFieldMetadata.name}`),
[ fieldMetadataInput.workspaceId,
{ [
name: computeObjectTargetTable(objectMetadata), {
action: 'alter', name: computeObjectTargetTable(objectMetadata),
columns: this.workspaceMigrationFactory.createColumnActions( action: 'alter',
WorkspaceMigrationColumnActionType.CREATE, columns: this.workspaceMigrationFactory.createColumnActions(
createdFieldMetadata, WorkspaceMigrationColumnActionType.CREATE,
), createdFieldMetadata,
} satisfies WorkspaceMigrationTableAction, ),
], } satisfies WorkspaceMigrationTableAction,
); ],
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
fieldMetadataInput.workspaceId, fieldMetadataInput.workspaceId,
); );
}
// TODO: Move viewField creation to a cdc scheduler // TODO: Move viewField creation to a cdc scheduler
const dataSourceMetadata = const dataSourceMetadata =

5
packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/generate-nullable.ts
View File

@ -3,7 +3,12 @@ import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/fi
export function generateNullable( export function generateNullable(
type: FieldMetadataType, type: FieldMetadataType,
inputNullableValue?: boolean, inputNullableValue?: boolean,
isRemoteCreation?: boolean,
): boolean { ): boolean {
if (isRemoteCreation) {
return true;
}
switch (type) { switch (type) {
case FieldMetadataType.TEXT: case FieldMetadataType.TEXT:
case FieldMetadataType.PHONE: case FieldMetadataType.PHONE:

13
packages/twenty-server/src/engine/metadata-modules/object-metadata/dtos/create-object.input.ts
View File

@ -1,7 +1,13 @@
import { Field, HideField, InputType } from '@nestjs/graphql'; import { Field, HideField, InputType } from '@nestjs/graphql';
import { BeforeCreateOne } from '@ptc-org/nestjs-query-graphql'; import { BeforeCreateOne } from '@ptc-org/nestjs-query-graphql';
import { IsNotEmpty, IsOptional, IsString, IsUUID } from 'class-validator'; import {
IsBoolean,
IsNotEmpty,
IsOptional,
IsString,
IsUUID,
} from 'class-validator';
import { IsValidMetadataName } from 'src/engine/decorators/metadata/is-valid-metadata-name.decorator'; import { IsValidMetadataName } from 'src/engine/decorators/metadata/is-valid-metadata-name.decorator';
import { BeforeCreateOneObject } from 'src/engine/metadata-modules/object-metadata/hooks/before-create-one-object.hook'; import { BeforeCreateOneObject } from 'src/engine/metadata-modules/object-metadata/hooks/before-create-one-object.hook';
@ -56,4 +62,9 @@ export class CreateObjectInput {
@IsOptional() @IsOptional()
@Field({ nullable: true }) @Field({ nullable: true })
imageIdentifierFieldMetadataId?: string; imageIdentifierFieldMetadataId?: string;
@IsBoolean()
@IsOptional()
@Field({ nullable: true })
isRemote?: boolean;
} }

578
packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts
View File

@ -139,9 +139,11 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
}); });
} }
await this.relationMetadataRepository.delete( if (relationsToDelete.length > 0) {
relationsToDelete.map((relation) => relation.id), await this.relationMetadataRepository.delete(
); relationsToDelete.map((relation) => relation.id),
);
}
for (const relationToDelete of relationsToDelete) { for (const relationToDelete of relationsToDelete) {
const foreignKeyFieldsToDelete = await this.fieldMetadataRepository.find({ const foreignKeyFieldsToDelete = await this.fieldMetadataRepository.find({
@ -192,17 +194,19 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
await this.objectMetadataRepository.delete(objectMetadata.id); await this.objectMetadataRepository.delete(objectMetadata.id);
// DROP TABLE if (!objectMetadata.isRemote) {
await this.workspaceMigrationService.createCustomMigration( // DROP TABLE
generateMigrationName(`delete-${objectMetadata.nameSingular}`), await this.workspaceMigrationService.createCustomMigration(
workspaceId, generateMigrationName(`delete-${objectMetadata.nameSingular}`),
[ workspaceId,
{ [
name: computeObjectTargetTable(objectMetadata), {
action: 'drop', name: computeObjectTargetTable(objectMetadata),
}, action: 'drop',
], },
); ],
);
}
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId, workspaceId,
@ -233,292 +237,296 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
dataSourceId: lastDataSourceMetadata.id, dataSourceId: lastDataSourceMetadata.id,
targetTableName: 'DEPRECATED', targetTableName: 'DEPRECATED',
isActive: true, isActive: true,
isCustom: true, isCustom: !objectMetadataInput.isRemote,
isSystem: false, isSystem: false,
isRemote: false, isRemote: !!objectMetadataInput.isRemote,
fields: fields: !objectMetadataInput.isRemote
// Creating default fields. ? // Creating default fields.
// No need to create a custom migration for this though as the default columns are already // No need to create a custom migration for this though as the default columns are already
// created with default values which is not supported yet by workspace migrations. // created with default values which is not supported yet by workspace migrations.
[ [
{ {
standardId: baseObjectStandardFieldIds.id, standardId: baseObjectStandardFieldIds.id,
type: FieldMetadataType.UUID, type: FieldMetadataType.UUID,
name: 'id', name: 'id',
label: 'Id', label: 'Id',
targetColumnMap: { targetColumnMap: {
value: 'id', value: 'id',
},
icon: 'Icon123',
description: 'Id',
isNullable: false,
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'uuid',
}, },
icon: 'Icon123', {
description: 'Id', standardId: customObjectStandardFieldIds.name,
isNullable: false, type: FieldMetadataType.TEXT,
isActive: true, name: 'name',
isCustom: false, label: 'Name',
isSystem: true, targetColumnMap: {
workspaceId: objectMetadataInput.workspaceId, value: 'name',
defaultValue: 'uuid', },
}, icon: 'IconAbc',
{ description: 'Name',
standardId: customObjectStandardFieldIds.name, isNullable: false,
type: FieldMetadataType.TEXT, isActive: true,
name: 'name', isCustom: false,
label: 'Name', workspaceId: objectMetadataInput.workspaceId,
targetColumnMap: { defaultValue: "'Untitled'",
value: 'name',
}, },
icon: 'IconAbc', {
description: 'Name', standardId: baseObjectStandardFieldIds.createdAt,
isNullable: false, type: FieldMetadataType.DATE_TIME,
isActive: true, name: 'createdAt',
isCustom: false, label: 'Creation date',
workspaceId: objectMetadataInput.workspaceId, targetColumnMap: {
defaultValue: "'Untitled'", value: 'createdAt',
}, },
{ icon: 'IconCalendar',
standardId: baseObjectStandardFieldIds.createdAt, description: 'Creation date',
type: FieldMetadataType.DATE_TIME, isNullable: false,
name: 'createdAt', isActive: true,
label: 'Creation date', isCustom: false,
targetColumnMap: { workspaceId: objectMetadataInput.workspaceId,
value: 'createdAt', defaultValue: 'now',
}, },
icon: 'IconCalendar', {
description: 'Creation date', standardId: baseObjectStandardFieldIds.updatedAt,
isNullable: false, type: FieldMetadataType.DATE_TIME,
isActive: true, name: 'updatedAt',
isCustom: false, label: 'Update date',
workspaceId: objectMetadataInput.workspaceId, targetColumnMap: {
defaultValue: 'now', value: 'updatedAt',
}, },
{ icon: 'IconCalendar',
standardId: baseObjectStandardFieldIds.updatedAt, description: 'Update date',
type: FieldMetadataType.DATE_TIME, isNullable: false,
name: 'updatedAt', isActive: true,
label: 'Update date', isCustom: false,
targetColumnMap: { isSystem: true,
value: 'updatedAt', workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'now',
}, },
icon: 'IconCalendar', {
description: 'Update date', standardId: customObjectStandardFieldIds.position,
isNullable: false, type: FieldMetadataType.POSITION,
isActive: true, name: 'position',
isCustom: false, label: 'Position',
isSystem: true, targetColumnMap: {
workspaceId: objectMetadataInput.workspaceId, value: 'position',
defaultValue: 'now', },
}, icon: 'IconHierarchy2',
{ description: 'Position',
standardId: customObjectStandardFieldIds.position, isNullable: true,
type: FieldMetadataType.POSITION, isActive: true,
name: 'position', isCustom: false,
label: 'Position', isSystem: true,
targetColumnMap: { workspaceId: objectMetadataInput.workspaceId,
value: 'position', defaultValue: null,
}, },
icon: 'IconHierarchy2', ]
description: 'Position', : // No fields for remote objects.
isNullable: true, [],
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: null,
},
],
}); });
const { activityTargetObjectMetadata } = if (!objectMetadataInput.isRemote) {
await this.createActivityTargetRelation( const { eventObjectMetadata } = await this.createEventRelation(
objectMetadataInput.workspaceId, objectMetadataInput.workspaceId,
createdObjectMetadata, createdObjectMetadata,
); );
const { favoriteObjectMetadata } = await this.createFavoriteRelation( const { activityTargetObjectMetadata } =
objectMetadataInput.workspaceId, await this.createActivityTargetRelation(
createdObjectMetadata, objectMetadataInput.workspaceId,
); createdObjectMetadata,
);
const { attachmentObjectMetadata } = await this.createAttachmentRelation( const { favoriteObjectMetadata } = await this.createFavoriteRelation(
objectMetadataInput.workspaceId, objectMetadataInput.workspaceId,
createdObjectMetadata, createdObjectMetadata,
); );
const { eventObjectMetadata } = await this.createEventRelation( const { attachmentObjectMetadata } = await this.createAttachmentRelation(
objectMetadataInput.workspaceId, objectMetadataInput.workspaceId,
createdObjectMetadata, createdObjectMetadata,
); );
await this.workspaceMigrationService.createCustomMigration( await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdObjectMetadata.nameSingular}`), generateMigrationName(`create-${createdObjectMetadata.nameSingular}`),
createdObjectMetadata.workspaceId, createdObjectMetadata.workspaceId,
[ [
{ {
name: computeObjectTargetTable(createdObjectMetadata), name: computeObjectTargetTable(createdObjectMetadata),
action: 'create', action: 'create',
} satisfies WorkspaceMigrationTableAction, } satisfies WorkspaceMigrationTableAction,
// Add activity target relation // Add activity target relation
{ {
name: computeObjectTargetTable(activityTargetObjectMetadata), name: computeObjectTargetTable(activityTargetObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE, action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
columnType: 'uuid', columnType: 'uuid',
isNullable: true, isNullable: true,
} satisfies WorkspaceMigrationColumnCreate, } satisfies WorkspaceMigrationColumnCreate,
], ],
}, },
{ {
name: computeObjectTargetTable(activityTargetObjectMetadata), name: computeObjectTargetTable(activityTargetObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY, action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
referencedTableName: computeObjectTargetTable( referencedTableName: computeObjectTargetTable(
createdObjectMetadata, createdObjectMetadata,
), ),
referencedTableColumnName: 'id', referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE, onDelete: RelationOnDeleteAction.CASCADE,
}, },
], ],
}, },
// Add attachment relation // Add attachment relation
{ {
name: computeObjectTargetTable(attachmentObjectMetadata), name: computeObjectTargetTable(attachmentObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE, action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
columnType: 'uuid', columnType: 'uuid',
isNullable: true, isNullable: true,
} satisfies WorkspaceMigrationColumnCreate, } satisfies WorkspaceMigrationColumnCreate,
], ],
}, },
{ {
name: computeObjectTargetTable(attachmentObjectMetadata), name: computeObjectTargetTable(attachmentObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY, action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
referencedTableName: computeObjectTargetTable( referencedTableName: computeObjectTargetTable(
createdObjectMetadata, createdObjectMetadata,
), ),
referencedTableColumnName: 'id', referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE, onDelete: RelationOnDeleteAction.CASCADE,
}, },
], ],
}, },
// Add event relation // Add event relation
{ {
name: computeObjectTargetTable(eventObjectMetadata), name: computeObjectTargetTable(eventObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE, action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
columnType: 'uuid', columnType: 'uuid',
isNullable: true, isNullable: true,
} satisfies WorkspaceMigrationColumnCreate, } satisfies WorkspaceMigrationColumnCreate,
], ],
}, },
{ {
name: computeObjectTargetTable(eventObjectMetadata), name: computeObjectTargetTable(eventObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY, action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
referencedTableName: computeObjectTargetTable( referencedTableName: computeObjectTargetTable(
createdObjectMetadata, createdObjectMetadata,
), ),
referencedTableColumnName: 'id', referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE, onDelete: RelationOnDeleteAction.CASCADE,
}, },
], ],
}, },
// Add favorite relation // Add favorite relation
{ {
name: computeObjectTargetTable(favoriteObjectMetadata), name: computeObjectTargetTable(favoriteObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE, action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
columnType: 'uuid', columnType: 'uuid',
isNullable: true, isNullable: true,
} satisfies WorkspaceMigrationColumnCreate, } satisfies WorkspaceMigrationColumnCreate,
], ],
}, },
{ {
name: computeObjectTargetTable(favoriteObjectMetadata), name: computeObjectTargetTable(favoriteObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY, action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName( columnName: `${computeCustomName(
createdObjectMetadata.nameSingular, createdObjectMetadata.nameSingular,
false, false,
)}Id`, )}Id`,
referencedTableName: computeObjectTargetTable( referencedTableName: computeObjectTargetTable(
createdObjectMetadata, createdObjectMetadata,
), ),
referencedTableColumnName: 'id', referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE, onDelete: RelationOnDeleteAction.CASCADE,
}, },
], ],
}, },
{ {
name: computeObjectTargetTable(createdObjectMetadata), name: computeObjectTargetTable(createdObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE, action: WorkspaceMigrationColumnActionType.CREATE,
columnName: 'position', columnName: 'position',
columnType: 'float', columnType: 'float',
isNullable: true, isNullable: true,
} satisfies WorkspaceMigrationColumnCreate, } satisfies WorkspaceMigrationColumnCreate,
], ],
} satisfies WorkspaceMigrationTableAction, } satisfies WorkspaceMigrationTableAction,
// This is temporary until we implement mainIdentifier // This is temporary until we implement mainIdentifier
{ {
name: computeObjectTargetTable(createdObjectMetadata), name: computeObjectTargetTable(createdObjectMetadata),
action: 'alter', action: 'alter',
columns: [ columns: [
{ {
action: WorkspaceMigrationColumnActionType.CREATE, action: WorkspaceMigrationColumnActionType.CREATE,
columnName: 'name', columnName: 'name',
columnType: 'text', columnType: 'text',
defaultValue: "'Untitled'", defaultValue: "'Untitled'",
} satisfies WorkspaceMigrationColumnCreate, } satisfies WorkspaceMigrationColumnCreate,
], ],
} satisfies WorkspaceMigrationTableAction, } satisfies WorkspaceMigrationTableAction,
], ],
); );
}
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations( await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
createdObjectMetadata.workspaceId, createdObjectMetadata.workspaceId,

21
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input.ts Normal file
View File

@ -0,0 +1,21 @@
import { InputType, Field, ID } from '@nestjs/graphql';
import { IsEnum } from 'class-validator';
import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
@InputType()
export class RemoteTableInput {
@Field(() => ID)
remoteServerId: string;
@Field(() => String)
name: string;
@IsEnum(RemoteTableStatus)
@Field(() => RemoteTableStatus)
status: RemoteTableStatus;
@Field(() => String)
schema: string;
}

11
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { RemotePostgresTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [RemotePostgresTableService],
exports: [RemotePostgresTableService],
})
export class RemotePostgresTableModule {}

106
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service.ts Normal file
View File

@ -0,0 +1,106 @@
import { Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import {
buildPostgresUrl,
EXCLUDED_POSTGRES_SCHEMAS,
} from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
@Injectable()
export class RemotePostgresTableService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly environmentService: EnvironmentService,
) {}
public async findAvailableRemotePostgresTables(
workspaceId: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
) {
const remotePostgresTables =
await this.fetchTablesFromRemotePostgresSchema(remoteServer);
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
const currentForeignTableNames = (
await workspaceDataSource.query(
`SELECT foreign_table_name FROM information_schema.foreign_tables`,
)
).map((foreignTable) => foreignTable.foreign_table_name);
return remotePostgresTables.map((remoteTable) => ({
name: remoteTable.table_name,
schema: remoteTable.table_schema,
status: currentForeignTableNames.includes(remoteTable.table_name)
? RemoteTableStatus.SYNCED
: RemoteTableStatus.NOT_SYNCED,
}));
}
public async fetchPostgresTableColumnsSchema(
remoteServer: RemoteServerEntity<RemoteServerType>,
tableName: string,
tableSchema: string,
) {
const dataSource = new DataSource({
url: buildPostgresUrl(
this.environmentService.get('LOGIN_TOKEN_SECRET'),
remoteServer,
),
type: 'postgres',
logging: true,
});
await dataSource.initialize();
const columns = await dataSource.query(
`SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_name = '${tableName}' AND table_schema = '${tableSchema}'`,
);
await dataSource.destroy();
return columns;
}
private async fetchTablesFromRemotePostgresSchema(
remoteServer: RemoteServerEntity<RemoteServerType>,
) {
const dataSource = new DataSource({
url: buildPostgresUrl(
this.environmentService.get('LOGIN_TOKEN_SECRET'),
remoteServer,
),
type: 'postgres',
logging: true,
});
await dataSource.initialize();
const schemaNames = await dataSource.query(
`SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map(
(schema) => `'${schema}'`,
).join(', ')} ) order by schema_name limit 1`,
);
const remotePostgresTables = await dataSource.query(
`SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames
.map((schemaName) => `'${schemaName.schema_name}'`)
.join(', ')})`,
);
await dataSource.destroy();
return remotePostgresTables;
}
}

65
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util.ts Normal file
View File

@ -0,0 +1,65 @@
import { Repository } from 'typeorm/repository/Repository';
import { decryptText } from 'src/engine/core-modules/auth/auth.util';
import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
export const EXCLUDED_POSTGRES_SCHEMAS = [
'information_schema',
'pg_catalog',
'pg_toast',
];
export const buildPostgresUrl = (
secretKey: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
): string => {
const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions;
const userMappingOptions = remoteServer.userMappingOptions;
const password = decryptText(userMappingOptions.password, secretKey);
const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`;
return url;
};
export const mapUdtNameToFieldType = (udtName: string): FieldMetadataType => {
switch (udtName) {
case 'uuid':
return FieldMetadataType.UUID;
case 'varchar':
return FieldMetadataType.TEXT;
case 'bool':
return FieldMetadataType.BOOLEAN;
case 'timestamp':
case 'timestamptz':
return FieldMetadataType.DATE_TIME;
default:
return FieldMetadataType.TEXT;
}
};
export const isPostgreSQLIntegrationEnabled = async (
featureFlagRepository: Repository<FeatureFlagEntity>,
workspaceId: string,
) => {
const featureFlag = await featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKeys.IsPostgreSQLIntegrationEnabled,
value: true,
});
const featureFlagEnabled = featureFlag && featureFlag.value;
if (!featureFlagEnabled) {
throw new Error('PostgreSQL integration is not enabled');
}
};

12
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts
View File

@ -1,15 +1,27 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity'; import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemotePostgresTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module';
import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver'; import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service'; import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({ @Module({
imports: [ imports: [
TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'), TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
WorkspaceDataSourceModule, WorkspaceDataSourceModule,
DataSourceModule,
ObjectMetadataModule,
FieldMetadataModule,
RemotePostgresTableModule,
WorkspaceCacheVersionModule,
], ],
providers: [RemoteTableService, RemoteTableResolver], providers: [RemoteTableService, RemoteTableResolver],
}) })

14
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts
View File

@ -1,10 +1,11 @@
import { UseGuards } from '@nestjs/common'; import { UseGuards } from '@nestjs/common';
import { Args, Query, Resolver } from '@nestjs/graphql'; import { Args, Mutation, Query, Resolver } from '@nestjs/graphql';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input'; import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import { RemoteTableDTO } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; import { RemoteTableDTO } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service'; import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
@ -23,4 +24,15 @@ export class RemoteTableResolver {
workspaceId, workspaceId,
); );
} }
@Mutation(() => RemoteTableDTO)
async updateRemoteTableSyncStatus(
@Args('input') input: RemoteTableInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteTableService.updateRemoteTableSyncStatus(
input,
workspaceId,
);
}
} }

219
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts
View File

@ -8,12 +8,23 @@ import {
RemoteServerEntity, RemoteServerEntity,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity'; } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto'; import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { import {
EXCLUDED_POSTGRES_SCHEMAS, isPostgreSQLIntegrationEnabled,
buildPostgresUrl, mapUdtNameToFieldType,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util'; } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { RemotePostgresTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service';
import { snakeCase } from 'src/utils/snake-case';
import { capitalize } from 'src/utils/capitalize';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
export class RemoteTableService { export class RemoteTableService {
constructor( constructor(
@ -21,8 +32,14 @@ export class RemoteTableService {
private readonly remoteServerRepository: Repository< private readonly remoteServerRepository: Repository<
RemoteServerEntity<RemoteServerType> RemoteServerEntity<RemoteServerType>
>, >,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
private readonly workspaceDataSourceService: WorkspaceDataSourceService, private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly environmentService: EnvironmentService, private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly dataSourceService: DataSourceService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly fieldMetadataService: FieldMetadataService,
private readonly remotePostgresTableService: RemotePostgresTableService,
) {} ) {}
public async findAvailableRemoteTablesByServerId( public async findAvailableRemoteTablesByServerId(
@ -42,7 +59,12 @@ export class RemoteTableService {
switch (remoteServer.foreignDataWrapperType) { switch (remoteServer.foreignDataWrapperType) {
case RemoteServerType.POSTGRES_FDW: case RemoteServerType.POSTGRES_FDW:
return this.findAvailableRemotePostgresTables( await isPostgreSQLIntegrationEnabled(
this.featureFlagRepository,
workspaceId,
);
return this.remotePostgresTableService.findAvailableRemotePostgresTables(
workspaceId, workspaceId,
remoteServer, remoteServer,
); );
@ -51,62 +73,171 @@ export class RemoteTableService {
} }
} }
// TODO: may be moved into a separated postgres table service once we have more use cases public async updateRemoteTableSyncStatus(
private async findAvailableRemotePostgresTables( input: RemoteTableInput,
workspaceId: string, workspaceId: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
) { ) {
const remotePostgresTables = const remoteServer = await this.remoteServerRepository.findOne({
await this.fetchTablesFromRemotePostgresSchema(remoteServer); where: {
id: input.remoteServerId,
workspaceId,
},
});
if (!remoteServer) {
throw new NotFoundException('Remote server does not exist');
}
const dataSourcesMetatada =
await this.dataSourceService.getDataSourcesMetadataFromWorkspaceId(
workspaceId,
);
if (!dataSourcesMetatada) {
throw new NotFoundException('Workspace data source does not exist');
}
const workspaceDataSource = const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource( await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId, workspaceId,
); );
const currentForeignTableNames = ( switch (input.status) {
await workspaceDataSource.query( case RemoteTableStatus.SYNCED:
`SELECT foreign_table_name FROM information_schema.foreign_tables`, await this.buildForeignTableAndMetadata(
) input,
).map((foreignTable) => foreignTable.foreign_table_name); remoteServer,
workspaceId,
workspaceDataSource,
dataSourcesMetatada[0],
);
break;
case RemoteTableStatus.NOT_SYNCED:
await this.removeForeignTableAndMetadata(
input,
workspaceId,
workspaceDataSource,
dataSourcesMetatada[0].schema,
);
break;
default:
throw new Error('Unsupported remote table status');
}
return remotePostgresTables.map((remoteTable) => ({ await this.workspaceCacheVersionService.incrementVersion(workspaceId);
name: remoteTable.table_name,
schema: remoteTable.table_schema, return input;
status: currentForeignTableNames.includes(remoteTable.table_name)
? RemoteTableStatus.SYNCED
: RemoteTableStatus.NOT_SYNCED,
}));
} }
private async fetchTablesFromRemotePostgresSchema( private async buildForeignTableAndMetadata(
input: RemoteTableInput,
remoteServer: RemoteServerEntity<RemoteServerType>, remoteServer: RemoteServerEntity<RemoteServerType>,
workspaceId: string,
workspaceDataSource: DataSource,
dataSourceMetadata: DataSourceEntity,
) { ) {
const dataSource = new DataSource({ const localSchema = dataSourceMetadata.schema;
url: buildPostgresUrl(
this.environmentService.get('LOGIN_TOKEN_SECRET'),
remoteServer,
),
type: 'postgres',
logging: true,
});
await dataSource.initialize(); // TODO: Add strong typing for remote table columns. Will be done when we have another use case than Postgres
const remoteTableColumns = await this.fetchTableColumnsSchema(
const schemaNames = await dataSource.query( remoteServer,
`SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map( input.name,
(schema) => `'${schema}'`, input.schema,
).join(', ')} ) order by schema_name limit 1`,
); );
const remotePostgresTables = await dataSource.query( const foreignTableColumns = remoteTableColumns
`SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames .map((column) => `"${column.column_name}" ${column.data_type}`)
.map((schemaName) => `'${schemaName.schema_name}'`) .join(', ');
.join(', ')})`,
await workspaceDataSource.query(
`CREATE FOREIGN TABLE ${localSchema}."${input.name}Remote" (${foreignTableColumns}) SERVER "${remoteServer.foreignDataWrapperId}" OPTIONS (schema_name '${input.schema}', table_name '${input.name}')`,
);
await workspaceDataSource.query(
`COMMENT ON FOREIGN TABLE ${localSchema}."${input.name}Remote" IS e'@graphql({"primary_key_columns": ["id"], "totalCount": {"enabled": true}})'`,
); );
await dataSource.destroy(); // Should be done in a transaction. To be discussed
const objectMetadata = await this.objectMetadataService.createOne({
nameSingular: `${input.name}Remote`,
namePlural: `${input.name}Remotes`,
labelSingular: `${capitalize(snakeCase(input.name)).replace(
/_/g,
' ',
)} remote`,
labelPlural: `${capitalize(snakeCase(input.name)).replace(
/_/g,
' ',
)} remotes`,
description: 'Remote table',
dataSourceId: dataSourceMetadata.id,
workspaceId: workspaceId,
icon: 'IconUser',
isRemote: true,
} as CreateObjectInput);
return remotePostgresTables; for (const column of remoteTableColumns) {
const field = await this.fieldMetadataService.createOne({
name: column.column_name,
label: capitalize(snakeCase(column.column_name)).replace(/_/g, ' '),
description: 'Field of remote',
// TODO: function should work for other types than Postgres
type: mapUdtNameToFieldType(column.udt_name),
workspaceId: workspaceId,
objectMetadataId: objectMetadata.id,
isRemoteCreation: true,
isNullable: true,
} as CreateFieldInput);
if (column.column_name === 'id') {
await this.objectMetadataService.updateOne(objectMetadata.id, {
labelIdentifierFieldMetadataId: field.id,
});
}
}
}
private async removeForeignTableAndMetadata(
input: RemoteTableInput,
workspaceId: string,
workspaceDataSource: DataSource,
localSchema: string,
) {
const objectMetadata =
await this.objectMetadataService.findOneWithinWorkspace(workspaceId, {
where: { nameSingular: `${input.name}Remote` },
});
if (objectMetadata) {
await this.objectMetadataService.deleteOneObject(
{ id: objectMetadata.id },
workspaceId,
);
}
await workspaceDataSource.query(
`DROP FOREIGN TABLE ${localSchema}."${input.name}Remote"`,
);
}
private async fetchTableColumnsSchema(
remoteServer: RemoteServerEntity<RemoteServerType>,
tableName: string,
tableSchema: string,
) {
switch (remoteServer.foreignDataWrapperType) {
case RemoteServerType.POSTGRES_FDW:
await isPostgreSQLIntegrationEnabled(
this.featureFlagRepository,
remoteServer.workspaceId,
);
return this.remotePostgresTableService.fetchPostgresTableColumnsSchema(
remoteServer,
tableName,
tableSchema,
);
default:
throw new Error('Unsupported foreign data wrapper type');
}
} }
} }

25
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts
View File

@ -1,25 +0,0 @@
import { decryptText } from 'src/engine/core-modules/auth/auth.util';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
export const EXCLUDED_POSTGRES_SCHEMAS = [
'information_schema',
'pg_catalog',
'pg_toast',
];
export const buildPostgresUrl = (
secretKey: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
): string => {
const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions;
const userMappingOptions = remoteServer.userMappingOptions;
const password = decryptText(userMappingOptions.password, secretKey);
const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`;
return url;
};