admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Sync remote object (#4713)

Browse Source 
* Sync objects

* Generate data for isRemote

* Add cache version update

* Add label identifier + fix field metadata input

---------

Co-authored-by: Thomas Trompette <thomast@twenty.com>
This commit is contained in:
Thomas Trompette
2024-03-29 18:23:58 +01:00
committed by GitHub
parent 7f3623239a
commit 1d351a29b8
22 changed files with 777 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/twenty-front/src/generated-metadata/gql.ts
View File

@ -20,7 +20,7 @@ const documents = {
"\n mutation UpdateOneObjectMetadataItem(\n $idToUpdate: ID!\n $updatePayload: UpdateObjectInput!\n ) {\n updateOneObject(input: { id: $idToUpdate, update: $updatePayload }) {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n }\n }\n": types.UpdateOneObjectMetadataItemDocument,
"\n mutation DeleteOneObjectMetadataItem($idToDelete: ID!) {\n deleteOneObject(input: { id: $idToDelete }) {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n }\n }\n": types.DeleteOneObjectMetadataItemDocument,
"\n mutation DeleteOneFieldMetadataItem($idToDelete: ID!) {\n deleteOneField(input: { id: $idToDelete }) {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isNullable\n createdAt\n updatedAt\n }\n }\n": types.DeleteOneFieldMetadataItemDocument,
"\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n": types.ObjectMetadataItemsDocument,
"\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isRemote\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n": types.ObjectMetadataItemsDocument,
};
/**
@ -68,7 +68,7 @@ export function graphql(source: "\n mutation DeleteOneFieldMetadataItem($idToDe
/**
* The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/
export function graphql(source: "\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"): (typeof documents)["\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"];
export function graphql(source: "\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isRemote\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"): (typeof documents)["\n query ObjectMetadataItems(\n $objectFilter: objectFilter\n $fieldFilter: fieldFilter\n ) {\n objects(paging: { first: 1000 }, filter: $objectFilter) {\n edges {\n node {\n id\n dataSourceId\n nameSingular\n namePlural\n labelSingular\n labelPlural\n description\n icon\n isCustom\n isRemote\n isActive\n isSystem\n createdAt\n updatedAt\n labelIdentifierFieldMetadataId\n imageIdentifierFieldMetadataId\n fields(paging: { first: 1000 }, filter: $fieldFilter) {\n edges {\n node {\n id\n type\n name\n label\n description\n icon\n isCustom\n isActive\n isSystem\n isNullable\n createdAt\n updatedAt\n fromRelationMetadata {\n id\n relationType\n toObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n toFieldMetadataId\n }\n toRelationMetadata {\n id\n relationType\n fromObjectMetadata {\n id\n dataSourceId\n nameSingular\n namePlural\n isSystem\n }\n fromFieldMetadataId\n }\n defaultValue\n options\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n }\n pageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n }\n }\n }\n"];
export function graphql(source: string) {
return (documents as any)[source] ?? {};

21
packages/twenty-front/src/generated-metadata/graphql.ts
View File

File diff suppressed because one or more lines are too long

1
packages/twenty-front/src/modules/object-metadata/graphql/queries.ts
View File

@ -17,6 +17,7 @@ export const FIND_MANY_OBJECT_METADATA_ITEMS = gql`
description
icon
isCustom
isRemote
isActive
isSystem
createdAt

1
packages/twenty-front/src/modules/object-metadata/hooks/__mocks__/useFindManyObjectMetadataItems.ts
View File

@ -17,6 +17,7 @@ export const query = gql`
description
icon
isCustom
isRemote
isActive
isSystem
createdAt

7
packages/twenty-front/src/modules/object-metadata/utils/getObjectMetadataItemsMock.ts
View File

@ -13,6 +13,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A webhook',
icon: 'IconRobot',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z',
@ -30,6 +31,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'An api key',
icon: 'IconRobot',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z',
@ -150,6 +152,7 @@ export const getObjectMetadataItemsMock = () => {
description: '(System) View Sorts',
icon: 'IconArrowsSort',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z',
@ -282,6 +285,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A calendar event',
icon: 'IconCalendarEvent',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z',
@ -299,6 +303,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'An opportunity',
icon: 'IconTargetArrow',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: false,
createdAt: '2023-11-30T11:13:15.206Z',
@ -617,6 +622,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A person',
icon: 'IconUser',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: false,
createdAt: '2023-11-30T11:13:15.206Z',
@ -1013,6 +1019,7 @@ export const getObjectMetadataItemsMock = () => {
description: 'A workspace member',
icon: 'IconUserCircle',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: true,
createdAt: '2023-11-30T11:13:15.206Z',

1
packages/twenty-front/src/modules/object-metadata/validation-schemas/objectMetadataItemSchema.ts
View File

@ -15,6 +15,7 @@ export const objectMetadataItemSchema = z.object({
imageIdentifierFieldMetadataId: z.string().uuid().nullable(),
isActive: z.boolean(),
isCustom: z.boolean(),
isRemote: z.boolean(),
isSystem: z.boolean(),
labelIdentifierFieldMetadataId: z.string().uuid().nullable(),
labelPlural: z.string().trim().min(1),

4
packages/twenty-front/src/modules/object-record/record-index/hooks/useLoadRecordIndexTable.ts
View File

@ -29,6 +29,10 @@ export const useFindManyParams = (
objectMetadataItem?.fields ?? [],
);
if (objectMetadataItem?.isRemote) {
return { objectNameSingular, filter };
}
const orderBy = turnSortsIntoOrderBy(
tableSorts,
objectMetadataItem?.fields ?? [],

2
packages/twenty-front/src/testing/mock-data/metadata.ts
View File

@ -18,6 +18,7 @@ export const mockedPeopleMetadata = {
description: 'A person',
icon: 'IconUser',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: false,
createdAt: '2023-12-15T15:29:39.070Z',
@ -594,6 +595,7 @@ export const mockedCompaniesMetadata = {
description: 'A company',
icon: 'IconBuildingSkyscraper',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: false,
createdAt: '2023-12-15T15:29:39.070Z',

1
packages/twenty-front/src/testing/mock-data/objectMetadataItems.ts
View File

@ -11,6 +11,7 @@ export const mockObjectMetadataItem: ObjectMetadataItem = {
description: 'A company',
icon: 'IconBuildingSkyscraper',
isCustom: false,
isRemote: false,
isActive: true,
isSystem: false,
createdAt: '2023-12-19T12:15:28.459Z',

6
packages/twenty-server/src/engine/metadata-modules/field-metadata/dtos/create-field.input.ts
View File

@ -1,6 +1,6 @@
import { Field, InputType, OmitType } from '@nestjs/graphql';
import { IsUUID, ValidateNested } from 'class-validator';
import { IsOptional, IsUUID, ValidateNested } from 'class-validator';
import { Type } from 'class-transformer';
import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto';
@ -14,6 +14,10 @@ export class CreateFieldInput extends OmitType(
@IsUUID()
@Field()
objectMetadataId: string;
@Field(() => Boolean, { nullable: true })
@IsOptional()
isRemoteCreation?: boolean;
}
@InputType()

39
packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts
View File

@ -122,12 +122,13 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
...fieldMetadataInput,
targetColumnMap: generateTargetColumnMap(
fieldMetadataInput.type,
true,
!fieldMetadataInput.isRemoteCreation,
fieldMetadataInput.name,
),
isNullable: generateNullable(
fieldMetadataInput.type,
fieldMetadataInput.isNullable,
fieldMetadataInput.isRemoteCreation,
),
defaultValue:
fieldMetadataInput.defaultValue ??
@ -142,24 +143,26 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadataEntit
isCustom: true,
});
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdFieldMetadata.name}`),
fieldMetadataInput.workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
action: 'alter',
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
createdFieldMetadata,
),
} satisfies WorkspaceMigrationTableAction,
],
);
if (!fieldMetadataInput.isRemoteCreation) {
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdFieldMetadata.name}`),
fieldMetadataInput.workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
action: 'alter',
columns: this.workspaceMigrationFactory.createColumnActions(
WorkspaceMigrationColumnActionType.CREATE,
createdFieldMetadata,
),
} satisfies WorkspaceMigrationTableAction,
],
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
fieldMetadataInput.workspaceId,
);
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
fieldMetadataInput.workspaceId,
);
}
// TODO: Move viewField creation to a cdc scheduler
const dataSourceMetadata =

5
packages/twenty-server/src/engine/metadata-modules/field-metadata/utils/generate-nullable.ts
View File

@ -3,7 +3,12 @@ import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/fi
export function generateNullable(
type: FieldMetadataType,
inputNullableValue?: boolean,
isRemoteCreation?: boolean,
): boolean {
if (isRemoteCreation) {
return true;
}
switch (type) {
case FieldMetadataType.TEXT:
case FieldMetadataType.PHONE:

13
packages/twenty-server/src/engine/metadata-modules/object-metadata/dtos/create-object.input.ts
View File

@ -1,7 +1,13 @@
import { Field, HideField, InputType } from '@nestjs/graphql';
import { BeforeCreateOne } from '@ptc-org/nestjs-query-graphql';
import { IsNotEmpty, IsOptional, IsString, IsUUID } from 'class-validator';
import {
IsBoolean,
IsNotEmpty,
IsOptional,
IsString,
IsUUID,
} from 'class-validator';
import { IsValidMetadataName } from 'src/engine/decorators/metadata/is-valid-metadata-name.decorator';
import { BeforeCreateOneObject } from 'src/engine/metadata-modules/object-metadata/hooks/before-create-one-object.hook';
@ -56,4 +62,9 @@ export class CreateObjectInput {
@IsOptional()
@Field({ nullable: true })
imageIdentifierFieldMetadataId?: string;
@IsBoolean()
@IsOptional()
@Field({ nullable: true })
isRemote?: boolean;
}

578
packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.service.ts
View File

@ -139,9 +139,11 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
});
}
await this.relationMetadataRepository.delete(
relationsToDelete.map((relation) => relation.id),
);
if (relationsToDelete.length > 0) {
await this.relationMetadataRepository.delete(
relationsToDelete.map((relation) => relation.id),
);
}
for (const relationToDelete of relationsToDelete) {
const foreignKeyFieldsToDelete = await this.fieldMetadataRepository.find({
@ -192,17 +194,19 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
await this.objectMetadataRepository.delete(objectMetadata.id);
// DROP TABLE
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`delete-${objectMetadata.nameSingular}`),
workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
action: 'drop',
},
],
);
if (!objectMetadata.isRemote) {
// DROP TABLE
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`delete-${objectMetadata.nameSingular}`),
workspaceId,
[
{
name: computeObjectTargetTable(objectMetadata),
action: 'drop',
},
],
);
}
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId,
@ -233,292 +237,296 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadataEnt
dataSourceId: lastDataSourceMetadata.id,
targetTableName: 'DEPRECATED',
isActive: true,
isCustom: true,
isCustom: !objectMetadataInput.isRemote,
isSystem: false,
isRemote: false,
fields:
// Creating default fields.
// No need to create a custom migration for this though as the default columns are already
// created with default values which is not supported yet by workspace migrations.
[
{
standardId: baseObjectStandardFieldIds.id,
type: FieldMetadataType.UUID,
name: 'id',
label: 'Id',
targetColumnMap: {
value: 'id',
isRemote: !!objectMetadataInput.isRemote,
fields: !objectMetadataInput.isRemote
? // Creating default fields.
// No need to create a custom migration for this though as the default columns are already
// created with default values which is not supported yet by workspace migrations.
[
{
standardId: baseObjectStandardFieldIds.id,
type: FieldMetadataType.UUID,
name: 'id',
label: 'Id',
targetColumnMap: {
value: 'id',
},
icon: 'Icon123',
description: 'Id',
isNullable: false,
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'uuid',
},
icon: 'Icon123',
description: 'Id',
isNullable: false,
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'uuid',
},
{
standardId: customObjectStandardFieldIds.name,
type: FieldMetadataType.TEXT,
name: 'name',
label: 'Name',
targetColumnMap: {
value: 'name',
{
standardId: customObjectStandardFieldIds.name,
type: FieldMetadataType.TEXT,
name: 'name',
label: 'Name',
targetColumnMap: {
value: 'name',
},
icon: 'IconAbc',
description: 'Name',
isNullable: false,
isActive: true,
isCustom: false,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: "'Untitled'",
},
icon: 'IconAbc',
description: 'Name',
isNullable: false,
isActive: true,
isCustom: false,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: "'Untitled'",
},
{
standardId: baseObjectStandardFieldIds.createdAt,
type: FieldMetadataType.DATE_TIME,
name: 'createdAt',
label: 'Creation date',
targetColumnMap: {
value: 'createdAt',
{
standardId: baseObjectStandardFieldIds.createdAt,
type: FieldMetadataType.DATE_TIME,
name: 'createdAt',
label: 'Creation date',
targetColumnMap: {
value: 'createdAt',
},
icon: 'IconCalendar',
description: 'Creation date',
isNullable: false,
isActive: true,
isCustom: false,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'now',
},
icon: 'IconCalendar',
description: 'Creation date',
isNullable: false,
isActive: true,
isCustom: false,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'now',
},
{
standardId: baseObjectStandardFieldIds.updatedAt,
type: FieldMetadataType.DATE_TIME,
name: 'updatedAt',
label: 'Update date',
targetColumnMap: {
value: 'updatedAt',
{
standardId: baseObjectStandardFieldIds.updatedAt,
type: FieldMetadataType.DATE_TIME,
name: 'updatedAt',
label: 'Update date',
targetColumnMap: {
value: 'updatedAt',
},
icon: 'IconCalendar',
description: 'Update date',
isNullable: false,
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'now',
},
icon: 'IconCalendar',
description: 'Update date',
isNullable: false,
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: 'now',
},
{
standardId: customObjectStandardFieldIds.position,
type: FieldMetadataType.POSITION,
name: 'position',
label: 'Position',
targetColumnMap: {
value: 'position',
{
standardId: customObjectStandardFieldIds.position,
type: FieldMetadataType.POSITION,
name: 'position',
label: 'Position',
targetColumnMap: {
value: 'position',
},
icon: 'IconHierarchy2',
description: 'Position',
isNullable: true,
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: null,
},
icon: 'IconHierarchy2',
description: 'Position',
isNullable: true,
isActive: true,
isCustom: false,
isSystem: true,
workspaceId: objectMetadataInput.workspaceId,
defaultValue: null,
},
],
]
: // No fields for remote objects.
[],
});
const { activityTargetObjectMetadata } =
await this.createActivityTargetRelation(
if (!objectMetadataInput.isRemote) {
const { eventObjectMetadata } = await this.createEventRelation(
objectMetadataInput.workspaceId,
createdObjectMetadata,
);
const { favoriteObjectMetadata } = await this.createFavoriteRelation(
objectMetadataInput.workspaceId,
createdObjectMetadata,
);
const { activityTargetObjectMetadata } =
await this.createActivityTargetRelation(
objectMetadataInput.workspaceId,
createdObjectMetadata,
);
const { attachmentObjectMetadata } = await this.createAttachmentRelation(
objectMetadataInput.workspaceId,
createdObjectMetadata,
);
const { favoriteObjectMetadata } = await this.createFavoriteRelation(
objectMetadataInput.workspaceId,
createdObjectMetadata,
);
const { eventObjectMetadata } = await this.createEventRelation(
objectMetadataInput.workspaceId,
createdObjectMetadata,
);
const { attachmentObjectMetadata } = await this.createAttachmentRelation(
objectMetadataInput.workspaceId,
createdObjectMetadata,
);
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdObjectMetadata.nameSingular}`),
createdObjectMetadata.workspaceId,
[
{
name: computeObjectTargetTable(createdObjectMetadata),
action: 'create',
} satisfies WorkspaceMigrationTableAction,
// Add activity target relation
{
name: computeObjectTargetTable(activityTargetObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(activityTargetObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
// Add attachment relation
{
name: computeObjectTargetTable(attachmentObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(attachmentObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
// Add event relation
{
name: computeObjectTargetTable(eventObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(eventObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
// Add favorite relation
{
name: computeObjectTargetTable(favoriteObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(favoriteObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
{
name: computeObjectTargetTable(createdObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: 'position',
columnType: 'float',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
} satisfies WorkspaceMigrationTableAction,
// This is temporary until we implement mainIdentifier
{
name: computeObjectTargetTable(createdObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: 'name',
columnType: 'text',
defaultValue: "'Untitled'",
} satisfies WorkspaceMigrationColumnCreate,
],
} satisfies WorkspaceMigrationTableAction,
],
);
await this.workspaceMigrationService.createCustomMigration(
generateMigrationName(`create-${createdObjectMetadata.nameSingular}`),
createdObjectMetadata.workspaceId,
[
{
name: computeObjectTargetTable(createdObjectMetadata),
action: 'create',
} satisfies WorkspaceMigrationTableAction,
// Add activity target relation
{
name: computeObjectTargetTable(activityTargetObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(activityTargetObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
// Add attachment relation
{
name: computeObjectTargetTable(attachmentObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(attachmentObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
// Add event relation
{
name: computeObjectTargetTable(eventObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(eventObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
// Add favorite relation
{
name: computeObjectTargetTable(favoriteObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
columnType: 'uuid',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
},
{
name: computeObjectTargetTable(favoriteObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE_FOREIGN_KEY,
columnName: `${computeCustomName(
createdObjectMetadata.nameSingular,
false,
)}Id`,
referencedTableName: computeObjectTargetTable(
createdObjectMetadata,
),
referencedTableColumnName: 'id',
onDelete: RelationOnDeleteAction.CASCADE,
},
],
},
{
name: computeObjectTargetTable(createdObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: 'position',
columnType: 'float',
isNullable: true,
} satisfies WorkspaceMigrationColumnCreate,
],
} satisfies WorkspaceMigrationTableAction,
// This is temporary until we implement mainIdentifier
{
name: computeObjectTargetTable(createdObjectMetadata),
action: 'alter',
columns: [
{
action: WorkspaceMigrationColumnActionType.CREATE,
columnName: 'name',
columnType: 'text',
defaultValue: "'Untitled'",
} satisfies WorkspaceMigrationColumnCreate,
],
} satisfies WorkspaceMigrationTableAction,
],
);
}
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
createdObjectMetadata.workspaceId,

21
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input.ts Normal file
View File

@ -0,0 +1,21 @@
import { InputType, Field, ID } from '@nestjs/graphql';
import { IsEnum } from 'class-validator';
import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
@InputType()
export class RemoteTableInput {
@Field(() => ID)
remoteServerId: string;
@Field(() => String)
name: string;
@IsEnum(RemoteTableStatus)
@Field(() => RemoteTableStatus)
status: RemoteTableStatus;
@Field(() => String)
schema: string;
}

11
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module.ts Normal file
View File

@ -0,0 +1,11 @@
import { Module } from '@nestjs/common';
import { RemotePostgresTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
imports: [WorkspaceDataSourceModule],
providers: [RemotePostgresTableService],
exports: [RemotePostgresTableService],
})
export class RemotePostgresTableModule {}

106
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service.ts Normal file
View File

@ -0,0 +1,106 @@
import { Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import {
buildPostgresUrl,
EXCLUDED_POSTGRES_SCHEMAS,
} from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
@Injectable()
export class RemotePostgresTableService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly environmentService: EnvironmentService,
) {}
public async findAvailableRemotePostgresTables(
workspaceId: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
) {
const remotePostgresTables =
await this.fetchTablesFromRemotePostgresSchema(remoteServer);
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
const currentForeignTableNames = (
await workspaceDataSource.query(
`SELECT foreign_table_name FROM information_schema.foreign_tables`,
)
).map((foreignTable) => foreignTable.foreign_table_name);
return remotePostgresTables.map((remoteTable) => ({
name: remoteTable.table_name,
schema: remoteTable.table_schema,
status: currentForeignTableNames.includes(remoteTable.table_name)
? RemoteTableStatus.SYNCED
: RemoteTableStatus.NOT_SYNCED,
}));
}
public async fetchPostgresTableColumnsSchema(
remoteServer: RemoteServerEntity<RemoteServerType>,
tableName: string,
tableSchema: string,
) {
const dataSource = new DataSource({
url: buildPostgresUrl(
this.environmentService.get('LOGIN_TOKEN_SECRET'),
remoteServer,
),
type: 'postgres',
logging: true,
});
await dataSource.initialize();
const columns = await dataSource.query(
`SELECT column_name, data_type, udt_name FROM information_schema.columns WHERE table_name = '${tableName}' AND table_schema = '${tableSchema}'`,
);
await dataSource.destroy();
return columns;
}
private async fetchTablesFromRemotePostgresSchema(
remoteServer: RemoteServerEntity<RemoteServerType>,
) {
const dataSource = new DataSource({
url: buildPostgresUrl(
this.environmentService.get('LOGIN_TOKEN_SECRET'),
remoteServer,
),
type: 'postgres',
logging: true,
});
await dataSource.initialize();
const schemaNames = await dataSource.query(
`SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map(
(schema) => `'${schema}'`,
).join(', ')} ) order by schema_name limit 1`,
);
const remotePostgresTables = await dataSource.query(
`SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames
.map((schemaName) => `'${schemaName.schema_name}'`)
.join(', ')})`,
);
await dataSource.destroy();
return remotePostgresTables;
}
}

65
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util.ts Normal file
View File

@ -0,0 +1,65 @@
import { Repository } from 'typeorm/repository/Repository';
import { decryptText } from 'src/engine/core-modules/auth/auth.util';
import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
export const EXCLUDED_POSTGRES_SCHEMAS = [
'information_schema',
'pg_catalog',
'pg_toast',
];
export const buildPostgresUrl = (
secretKey: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
): string => {
const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions;
const userMappingOptions = remoteServer.userMappingOptions;
const password = decryptText(userMappingOptions.password, secretKey);
const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`;
return url;
};
export const mapUdtNameToFieldType = (udtName: string): FieldMetadataType => {
switch (udtName) {
case 'uuid':
return FieldMetadataType.UUID;
case 'varchar':
return FieldMetadataType.TEXT;
case 'bool':
return FieldMetadataType.BOOLEAN;
case 'timestamp':
case 'timestamptz':
return FieldMetadataType.DATE_TIME;
default:
return FieldMetadataType.TEXT;
}
};
export const isPostgreSQLIntegrationEnabled = async (
featureFlagRepository: Repository<FeatureFlagEntity>,
workspaceId: string,
) => {
const featureFlag = await featureFlagRepository.findOneBy({
workspaceId,
key: FeatureFlagKeys.IsPostgreSQLIntegrationEnabled,
value: true,
});
const featureFlagEnabled = featureFlag && featureFlag.value;
if (!featureFlagEnabled) {
throw new Error('PostgreSQL integration is not enabled');
}
};

12
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.module.ts
View File

@ -1,15 +1,27 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
import { RemoteServerEntity } from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemotePostgresTableModule } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.module';
import { RemoteTableResolver } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@Module({
imports: [
TypeOrmModule.forFeature([RemoteServerEntity], 'metadata'),
TypeOrmModule.forFeature([FeatureFlagEntity], 'core'),
WorkspaceDataSourceModule,
DataSourceModule,
ObjectMetadataModule,
FieldMetadataModule,
RemotePostgresTableModule,
WorkspaceCacheVersionModule,
],
providers: [RemoteTableService, RemoteTableResolver],
})

14
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.resolver.ts
View File

@ -1,10 +1,11 @@
import { UseGuards } from '@nestjs/common';
import { Args, Query, Resolver } from '@nestjs/graphql';
import { Args, Mutation, Query, Resolver } from '@nestjs/graphql';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { RemoteServerIdInput } from 'src/engine/metadata-modules/remote-server/dtos/remote-server-id.input';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import { RemoteTableDTO } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import { RemoteTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.service';
@ -23,4 +24,15 @@ export class RemoteTableResolver {
workspaceId,
);
}
@Mutation(() => RemoteTableDTO)
async updateRemoteTableSyncStatus(
@Args('input') input: RemoteTableInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
return this.remoteTableService.updateRemoteTableSyncStatus(
input,
workspaceId,
);
}
}

219
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/remote-table.service.ts
View File

@ -8,12 +8,23 @@ import {
RemoteServerEntity,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { RemoteTableStatus } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import {
EXCLUDED_POSTGRES_SCHEMAS,
buildPostgresUrl,
} from 'src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util';
isPostgreSQLIntegrationEnabled,
mapUdtNameToFieldType,
} from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/utils/remote-postgres-table.util';
import { RemoteTableInput } from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table-input';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
import { CreateObjectInput } from 'src/engine/metadata-modules/object-metadata/dtos/create-object.input';
import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service';
import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input';
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { RemotePostgresTableService } from 'src/engine/metadata-modules/remote-server/remote-table/remote-postgres-table/remote-postgres-table.service';
import { snakeCase } from 'src/utils/snake-case';
import { capitalize } from 'src/utils/capitalize';
import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service';
export class RemoteTableService {
constructor(
@ -21,8 +32,14 @@ export class RemoteTableService {
private readonly remoteServerRepository: Repository<
RemoteServerEntity<RemoteServerType>
>,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly environmentService: EnvironmentService,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
private readonly dataSourceService: DataSourceService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly fieldMetadataService: FieldMetadataService,
private readonly remotePostgresTableService: RemotePostgresTableService,
) {}
public async findAvailableRemoteTablesByServerId(
@ -42,7 +59,12 @@ export class RemoteTableService {
switch (remoteServer.foreignDataWrapperType) {
case RemoteServerType.POSTGRES_FDW:
return this.findAvailableRemotePostgresTables(
await isPostgreSQLIntegrationEnabled(
this.featureFlagRepository,
workspaceId,
);
return this.remotePostgresTableService.findAvailableRemotePostgresTables(
workspaceId,
remoteServer,
);
@ -51,62 +73,171 @@ export class RemoteTableService {
}
}
// TODO: may be moved into a separated postgres table service once we have more use cases
private async findAvailableRemotePostgresTables(
public async updateRemoteTableSyncStatus(
input: RemoteTableInput,
workspaceId: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
) {
const remotePostgresTables =
await this.fetchTablesFromRemotePostgresSchema(remoteServer);
const remoteServer = await this.remoteServerRepository.findOne({
where: {
id: input.remoteServerId,
workspaceId,
},
});
if (!remoteServer) {
throw new NotFoundException('Remote server does not exist');
}
const dataSourcesMetatada =
await this.dataSourceService.getDataSourcesMetadataFromWorkspaceId(
workspaceId,
);
if (!dataSourcesMetatada) {
throw new NotFoundException('Workspace data source does not exist');
}
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
const currentForeignTableNames = (
await workspaceDataSource.query(
`SELECT foreign_table_name FROM information_schema.foreign_tables`,
)
).map((foreignTable) => foreignTable.foreign_table_name);
switch (input.status) {
case RemoteTableStatus.SYNCED:
await this.buildForeignTableAndMetadata(
input,
remoteServer,
workspaceId,
workspaceDataSource,
dataSourcesMetatada[0],
);
break;
case RemoteTableStatus.NOT_SYNCED:
await this.removeForeignTableAndMetadata(
input,
workspaceId,
workspaceDataSource,
dataSourcesMetatada[0].schema,
);
break;
default:
throw new Error('Unsupported remote table status');
}
return remotePostgresTables.map((remoteTable) => ({
name: remoteTable.table_name,
schema: remoteTable.table_schema,
status: currentForeignTableNames.includes(remoteTable.table_name)
? RemoteTableStatus.SYNCED
: RemoteTableStatus.NOT_SYNCED,
}));
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
return input;
}
private async fetchTablesFromRemotePostgresSchema(
private async buildForeignTableAndMetadata(
input: RemoteTableInput,
remoteServer: RemoteServerEntity<RemoteServerType>,
workspaceId: string,
workspaceDataSource: DataSource,
dataSourceMetadata: DataSourceEntity,
) {
const dataSource = new DataSource({
url: buildPostgresUrl(
this.environmentService.get('LOGIN_TOKEN_SECRET'),
remoteServer,
),
type: 'postgres',
logging: true,
});
const localSchema = dataSourceMetadata.schema;
await dataSource.initialize();
const schemaNames = await dataSource.query(
`SELECT schema_name FROM information_schema.schemata where schema_name not in ( ${EXCLUDED_POSTGRES_SCHEMAS.map(
(schema) => `'${schema}'`,
).join(', ')} ) order by schema_name limit 1`,
// TODO: Add strong typing for remote table columns. Will be done when we have another use case than Postgres
const remoteTableColumns = await this.fetchTableColumnsSchema(
remoteServer,
input.name,
input.schema,
);
const remotePostgresTables = await dataSource.query(
`SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema IN (${schemaNames
.map((schemaName) => `'${schemaName.schema_name}'`)
.join(', ')})`,
const foreignTableColumns = remoteTableColumns
.map((column) => `"${column.column_name}" ${column.data_type}`)
.join(', ');
await workspaceDataSource.query(
`CREATE FOREIGN TABLE ${localSchema}."${input.name}Remote" (${foreignTableColumns}) SERVER "${remoteServer.foreignDataWrapperId}" OPTIONS (schema_name '${input.schema}', table_name '${input.name}')`,
);
await workspaceDataSource.query(
`COMMENT ON FOREIGN TABLE ${localSchema}."${input.name}Remote" IS e'@graphql({"primary_key_columns": ["id"], "totalCount": {"enabled": true}})'`,
);
await dataSource.destroy();
// Should be done in a transaction. To be discussed
const objectMetadata = await this.objectMetadataService.createOne({
nameSingular: `${input.name}Remote`,
namePlural: `${input.name}Remotes`,
labelSingular: `${capitalize(snakeCase(input.name)).replace(
/_/g,
' ',
)} remote`,
labelPlural: `${capitalize(snakeCase(input.name)).replace(
/_/g,
' ',
)} remotes`,
description: 'Remote table',
dataSourceId: dataSourceMetadata.id,
workspaceId: workspaceId,
icon: 'IconUser',
isRemote: true,
} as CreateObjectInput);
return remotePostgresTables;
for (const column of remoteTableColumns) {
const field = await this.fieldMetadataService.createOne({
name: column.column_name,
label: capitalize(snakeCase(column.column_name)).replace(/_/g, ' '),
description: 'Field of remote',
// TODO: function should work for other types than Postgres
type: mapUdtNameToFieldType(column.udt_name),
workspaceId: workspaceId,
objectMetadataId: objectMetadata.id,
isRemoteCreation: true,
isNullable: true,
} as CreateFieldInput);
if (column.column_name === 'id') {
await this.objectMetadataService.updateOne(objectMetadata.id, {
labelIdentifierFieldMetadataId: field.id,
});
}
}
}
private async removeForeignTableAndMetadata(
input: RemoteTableInput,
workspaceId: string,
workspaceDataSource: DataSource,
localSchema: string,
) {
const objectMetadata =
await this.objectMetadataService.findOneWithinWorkspace(workspaceId, {
where: { nameSingular: `${input.name}Remote` },
});
if (objectMetadata) {
await this.objectMetadataService.deleteOneObject(
{ id: objectMetadata.id },
workspaceId,
);
}
await workspaceDataSource.query(
`DROP FOREIGN TABLE ${localSchema}."${input.name}Remote"`,
);
}
private async fetchTableColumnsSchema(
remoteServer: RemoteServerEntity<RemoteServerType>,
tableName: string,
tableSchema: string,
) {
switch (remoteServer.foreignDataWrapperType) {
case RemoteServerType.POSTGRES_FDW:
await isPostgreSQLIntegrationEnabled(
this.featureFlagRepository,
remoteServer.workspaceId,
);
return this.remotePostgresTableService.fetchPostgresTableColumnsSchema(
remoteServer,
tableName,
tableSchema,
);
default:
throw new Error('Unsupported foreign data wrapper type');
}
}
}

25
packages/twenty-server/src/engine/metadata-modules/remote-server/remote-table/utils/remote-table-postgres.util.ts
View File

@ -1,25 +0,0 @@
import { decryptText } from 'src/engine/core-modules/auth/auth.util';
import {
RemoteServerEntity,
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
export const EXCLUDED_POSTGRES_SCHEMAS = [
'information_schema',
'pg_catalog',
'pg_toast',
];
export const buildPostgresUrl = (
secretKey: string,
remoteServer: RemoteServerEntity<RemoteServerType>,
): string => {
const foreignDataWrapperOptions = remoteServer.foreignDataWrapperOptions;
const userMappingOptions = remoteServer.userMappingOptions;
const password = decryptText(userMappingOptions.password, secretKey);
const url = `postgres://${userMappingOptions.username}:${password}@${foreignDataWrapperOptions.host}:${foreignDataWrapperOptions.port}/${foreignDataWrapperOptions.dbname}`;
return url;
};