Minor refacto and fixes on Remotes updates (#5438)

In this PR

- Code refactoring
- v0 of adding "updates available" info in Connection sync status
<img width="835" alt="Capture d’écran 2024-05-16 à 17 02 07"
src="https://github.com/twentyhq/twenty/assets/51697796/9674d3ca-bed2-4520-a5a6-ba37bc242d06">

- fix distant table columns with not-camel case names are always
considered as new
This commit is contained in:
Marie
2024-05-16 17:31:34 +02:00
committed by GitHub
parent 9bc9513845
commit d741f4a5bd
15 changed files with 122 additions and 104 deletions

View File

@ -250,6 +250,14 @@ export type DeleteOneRelationInput = {
id: Scalars['UUID']['input'];
};
/** Schema update on a table */
export enum DistantTableUpdate {
ColumnsAdded = 'COLUMNS_ADDED',
ColumnsDeleted = 'COLUMNS_DELETED',
ColumnsTypeChanged = 'COLUMNS_TYPE_CHANGED',
TableDeleted = 'TABLE_DELETED'
}
export type EmailPasswordResetLink = {
__typename?: 'EmailPasswordResetLink';
/** Boolean that confirms query was dispatched */
@ -330,8 +338,6 @@ export enum FileFolder {
export type FindManyRemoteTablesInput = {
/** The id of the remote server. */
id: Scalars['ID']['input'];
/** Indicates if data from distant tables should be refreshed. */
refreshData?: InputMaybe<Scalars['Boolean']['input']>;
};
export type FullName = {
@ -812,7 +818,7 @@ export type RemoteTable = {
id?: Maybe<Scalars['UUID']['output']>;
name: Scalars['String']['output'];
schema?: Maybe<Scalars['String']['output']>;
schemaPendingUpdates?: Maybe<Array<TableUpdate>>;
schemaPendingUpdates?: Maybe<Array<DistantTableUpdate>>;
status: RemoteTableStatus;
};
@ -857,14 +863,6 @@ export type Support = {
supportFrontChatId?: Maybe<Scalars['String']['output']>;
};
/** Schema update on a table */
export enum TableUpdate {
ColumnsAdded = 'COLUMNS_ADDED',
ColumnsDeleted = 'COLUMNS_DELETED',
ColumnsTypeChanged = 'COLUMNS_TYPE_CHANGED',
TableDeleted = 'TABLE_DELETED'
}
export type Telemetry = {
__typename?: 'Telemetry';
anonymizationEnabled: Scalars['Boolean']['output'];
@ -1261,7 +1259,7 @@ export type RelationEdge = {
export type RemoteServerFieldsFragment = { __typename?: 'RemoteServer', id: string, createdAt: any, foreignDataWrapperId: string, foreignDataWrapperOptions?: any | null, foreignDataWrapperType: string, updatedAt: any, schema?: string | null, userMappingOptions?: { __typename?: 'UserMappingOptionsUser', user?: string | null } | null };
export type RemoteTableFieldsFragment = { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<TableUpdate> | null };
export type RemoteTableFieldsFragment = { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<DistantTableUpdate> | null };
export type CreateServerMutationVariables = Exact<{
input: CreateRemoteServerInput;
@ -1282,14 +1280,14 @@ export type SyncRemoteTableMutationVariables = Exact<{
}>;
export type SyncRemoteTableMutation = { __typename?: 'Mutation', syncRemoteTable: { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<TableUpdate> | null } };
export type SyncRemoteTableMutation = { __typename?: 'Mutation', syncRemoteTable: { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<DistantTableUpdate> | null } };
export type UnsyncRemoteTableMutationVariables = Exact<{
input: RemoteTableInput;
}>;
export type UnsyncRemoteTableMutation = { __typename?: 'Mutation', unsyncRemoteTable: { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<TableUpdate> | null } };
export type UnsyncRemoteTableMutation = { __typename?: 'Mutation', unsyncRemoteTable: { __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<DistantTableUpdate> | null } };
export type UpdateServerMutationVariables = Exact<{
input: UpdateRemoteServerInput;
@ -1310,7 +1308,7 @@ export type GetManyRemoteTablesQueryVariables = Exact<{
}>;
export type GetManyRemoteTablesQuery = { __typename?: 'Query', findAvailableRemoteTablesByServerId: Array<{ __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<TableUpdate> | null }> };
export type GetManyRemoteTablesQuery = { __typename?: 'Query', findAvailableRemoteTablesByServerId: Array<{ __typename?: 'RemoteTable', id?: any | null, name: string, schema?: string | null, status: RemoteTableStatus, schemaPendingUpdates?: Array<DistantTableUpdate> | null }> };
export type GetOneDatabaseConnectionQueryVariables = Exact<{
input: RemoteServerIdInput;

View File

@ -10,13 +10,11 @@ import {
type UseGetDatabaseConnectionTablesParams = {
connectionId: string;
skip?: boolean;
refreshData?: boolean;
};
export const useGetDatabaseConnectionTables = ({
connectionId,
skip,
refreshData,
}: UseGetDatabaseConnectionTablesParams) => {
const apolloMetadataClient = useApolloMetadataClient();
@ -29,7 +27,6 @@ export const useGetDatabaseConnectionTables = ({
variables: {
input: {
id: connectionId,
refreshData: refreshData,
},
},
});

View File

@ -25,13 +25,23 @@ export const SettingsIntegrationDatabaseConnectionSyncStatus = ({
(table) => table.status === RemoteTableStatus.Synced,
);
const updatesAvailable = tables.some(
(table) =>
table.schemaPendingUpdates?.length &&
table.schemaPendingUpdates.length > 0,
);
return (
<Status
color="green"
color={updatesAvailable ? 'yellow' : 'green'}
text={
syncedTables.length === 1
? `1 tracked table`
: `${syncedTables.length} tracked tables`
? `1 tracked table${
updatesAvailable ? ' (with pending schema updates)' : ''
}`
: `${syncedTables.length} tracked tables${
updatesAvailable ? ' (with pending schema updates)' : ''
}`
}
/>
);

View File

@ -7,9 +7,9 @@ import { useUnsyncRemoteTable } from '@/databases/hooks/useUnsyncRemoteTable';
import { SettingsListCard } from '@/settings/components/SettingsListCard';
import { SettingsIntegrationRemoteTableSyncStatusToggle } from '@/settings/integrations/components/SettingsIntegrationRemoteTableSyncStatusToggle';
import {
DistantTableUpdate,
RemoteTable,
RemoteTableStatus,
TableUpdate,
} from '~/generated-metadata/graphql';
import { isDefined } from '~/utils/isDefined';
@ -39,20 +39,22 @@ const StyledText = styled.h3`
margin: 0;
`;
const getTableUpdatesText = (schemaPendingUpdates: TableUpdate[]) => {
if (schemaPendingUpdates.includes(TableUpdate.TableDeleted)) {
const getDistantTableUpdatesText = (
schemaPendingUpdates: DistantTableUpdate[],
) => {
if (schemaPendingUpdates.includes(DistantTableUpdate.TableDeleted)) {
return 'Table has been deleted';
}
if (
schemaPendingUpdates.includes(TableUpdate.ColumnsAdded) &&
schemaPendingUpdates.includes(TableUpdate.ColumnsDeleted)
schemaPendingUpdates.includes(DistantTableUpdate.ColumnsAdded) &&
schemaPendingUpdates.includes(DistantTableUpdate.ColumnsDeleted)
) {
return 'Columns have been added and other deleted';
}
if (schemaPendingUpdates.includes(TableUpdate.ColumnsAdded)) {
if (schemaPendingUpdates.includes(DistantTableUpdate.ColumnsAdded)) {
return 'Columns have been added';
}
if (schemaPendingUpdates.includes(TableUpdate.ColumnsDeleted)) {
if (schemaPendingUpdates.includes(DistantTableUpdate.ColumnsDeleted)) {
return 'Columns have been deleted';
}
return null;
@ -71,7 +73,7 @@ export const SettingsIntegrationDatabaseTablesListCard = ({
...table,
id: table.name,
updatesText: table.schemaPendingUpdates
? getTableUpdatesText(table.schemaPendingUpdates)
? getDistantTableUpdatesText(table.schemaPendingUpdates)
: null,
})),
);

View File

@ -42,7 +42,6 @@ export const useDatabaseConnection = () => {
const { tables } = useGetDatabaseConnectionTables({
connectionId,
skip: !connection,
refreshData: true,
});
return { connection, integration, databaseKey, tables };

View File

@ -9,9 +9,9 @@ import {
RemoteServerType,
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column';
import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table';
import { STRIPE_DISTANT_TABLES } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/util/stripe-distant-tables.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
@Injectable()
export class DistantTableService {
@ -26,7 +26,7 @@ export class DistantTableService {
public getDistantTableColumns(
remoteServer: RemoteServerEntity<RemoteServerType>,
tableName: string,
): DistantTableColumn[] {
): PostgresTableSchemaColumn[] {
if (!remoteServer.availableTables) {
throw new BadRequestException(
'Remote server available tables are not defined',
@ -39,13 +39,8 @@ export class DistantTableService {
public async fetchDistantTables(
remoteServer: RemoteServerEntity<RemoteServerType>,
workspaceId: string,
refreshData?: boolean,
): Promise<DistantTables> {
if (!refreshData && remoteServer.availableTables) {
return remoteServer.availableTables;
}
return await this.createAvailableTables(remoteServer, workspaceId);
return this.createAvailableTables(remoteServer, workspaceId);
}
private async createAvailableTables(

View File

@ -1,6 +0,0 @@
// Type will evolve as we add more remote table types
export type DistantTableColumn = {
columnName: string;
dataType: string;
udtName: string;
};

View File

@ -1,5 +1,5 @@
import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
export type DistantTables = {
[tableName: string]: DistantTableColumn[];
[tableName: string]: PostgresTableSchemaColumn[];
};

View File

@ -1,17 +1,9 @@
import { InputType, ID, Field } from '@nestjs/graphql';
import { InputType, ID } from '@nestjs/graphql';
import { IDField } from '@ptc-org/nestjs-query-graphql';
import { IsOptional } from 'class-validator';
@InputType()
export class FindManyRemoteTablesInput {
@IDField(() => ID, { description: 'The id of the remote server.' })
id!: string;
@IsOptional()
@Field(() => Boolean, {
description: 'Indicates if data from distant tables should be refreshed.',
nullable: true,
})
refreshData?: boolean;
}

View File

@ -10,11 +10,11 @@ export enum RemoteTableStatus {
NOT_SYNCED = 'NOT_SYNCED',
}
export enum TableUpdate {
export enum DistantTableUpdate {
TABLE_DELETED = 'TABLE_DELETED',
COLUMNS_DELETED = 'COLUMN_DELETED',
COLUMNS_ADDED = 'COLUMN_ADDED',
COLUMNS_TYPE_CHANGED = 'COLUMN_TYPE_CHANGED',
COLUMNS_DELETED = 'COLUMNS_DELETED',
COLUMNS_ADDED = 'COLUMNS_ADDED',
COLUMNS_TYPE_CHANGED = 'COLUMNS_TYPE_CHANGED',
}
registerEnumType(RemoteTableStatus, {
@ -22,8 +22,8 @@ registerEnumType(RemoteTableStatus, {
description: 'Status of the table',
});
registerEnumType(TableUpdate, {
name: 'TableUpdate',
registerEnumType(DistantTableUpdate, {
name: 'DistantTableUpdate',
description: 'Schema update on a table',
});
@ -43,6 +43,6 @@ export class RemoteTableDTO {
schema?: string;
@IsOptional()
@Field(() => [TableUpdate], { nullable: true })
schemaPendingUpdates?: [TableUpdate];
@Field(() => [DistantTableUpdate], { nullable: true })
schemaPendingUpdates?: [DistantTableUpdate];
}

View File

@ -22,7 +22,6 @@ export class RemoteTableResolver {
return this.remoteTableService.findDistantTablesByServerId(
input.id,
workspaceId,
input.refreshData,
);
}

View File

@ -10,7 +10,7 @@ import {
} from 'src/engine/metadata-modules/remote-server/remote-server.entity';
import {
RemoteTableStatus,
TableUpdate,
DistantTableUpdate,
} from 'src/engine/metadata-modules/remote-server/remote-table/dtos/remote-table.dto';
import {
mapUdtNameToFieldType,
@ -38,9 +38,9 @@ import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/work
import { RemoteTableEntity } from 'src/engine/metadata-modules/remote-server/remote-table/remote-table.entity';
import { getRemoteTableLocalName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-remote-table-local-name.util';
import { DistantTableService } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/distant-table.service';
import { DistantTableColumn } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table-column';
import { DistantTables } from 'src/engine/metadata-modules/remote-server/remote-table/distant-table/types/distant-table';
import { RemoteTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/remote-table/types/remote-table-schema-column';
import { getForeignTableColumnName } from 'src/engine/metadata-modules/remote-server/remote-table/utils/get-foreign-table-column-name.util';
import { PostgresTableSchemaColumn } from 'src/engine/metadata-modules/remote-server/types/postgres-table-schema-column';
export class RemoteTableService {
private readonly logger = new Logger(RemoteTableService.name);
@ -62,11 +62,7 @@ export class RemoteTableService {
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
public async findDistantTablesByServerId(
id: string,
workspaceId: string,
refreshData?: boolean,
) {
public async findDistantTablesByServerId(id: string, workspaceId: string) {
const remoteServer = await this.remoteServerRepository.findOne({
where: {
id,
@ -90,10 +86,9 @@ export class RemoteTableService {
const distantTables = await this.distantTableService.fetchDistantTables(
remoteServer,
workspaceId,
refreshData,
);
if (!refreshData || currentRemoteTables.length === 0) {
if (currentRemoteTables.length === 0) {
const distantTablesWithStatus = Object.keys(distantTables).map(
(tableName) => ({
name: tableName,
@ -107,18 +102,41 @@ export class RemoteTableService {
return distantTablesWithStatus;
}
return this.getDistantTablesWithUpdates({
remoteServerSchema: remoteServer.schema,
workspaceId,
remoteTables: currentRemoteTables,
distantTables,
});
}
private async getDistantTablesWithUpdates({
remoteServerSchema,
workspaceId,
remoteTables,
distantTables,
}: {
remoteServerSchema: string;
workspaceId: string;
remoteTables: RemoteTableEntity[];
distantTables: DistantTables;
}) {
const schemaPendingUpdates =
await this.getSchemaUpdatesBetweenRemoteAndDistantTables({
await this.getSchemaUpdatesBetweenForeignAndDistantTables({
workspaceId,
remoteTables: currentRemoteTables,
remoteTables,
distantTables,
});
const remoteTablesDistantNames = remoteTables.map(
(remoteTable) => remoteTable.distantTableName,
);
const distantTablesWithUpdates = Object.keys(distantTables).map(
(tableName) => ({
name: tableName,
schema: remoteServer.schema,
status: currentRemoteTableDistantNames.includes(tableName)
schema: remoteServerSchema,
status: remoteTablesDistantNames.includes(tableName)
? RemoteTableStatus.SYNCED
: RemoteTableStatus.NOT_SYNCED,
schemaPendingUpdates: schemaPendingUpdates[tableName],
@ -127,11 +145,11 @@ export class RemoteTableService {
const deletedTables = Object.entries(schemaPendingUpdates)
.filter(([_tableName, updates]) =>
updates.includes(TableUpdate.TABLE_DELETED),
updates.includes(DistantTableUpdate.TABLE_DELETED),
)
.map(([tableName, updates]) => ({
name: tableName,
schema: remoteServer.schema,
schema: remoteServerSchema,
status: RemoteTableStatus.SYNCED,
schemaPendingUpdates: updates,
}));
@ -139,7 +157,7 @@ export class RemoteTableService {
return distantTablesWithUpdates.concat(deletedTables);
}
private async getSchemaUpdatesBetweenRemoteAndDistantTables({
private async getSchemaUpdatesBetweenForeignAndDistantTables({
workspaceId,
remoteTables,
distantTables,
@ -147,7 +165,7 @@ export class RemoteTableService {
workspaceId: string;
remoteTables: RemoteTableEntity[];
distantTables: DistantTables;
}): Promise<{ [tablename: string]: TableUpdate[] }> {
}): Promise<{ [tablename: string]: DistantTableUpdate[] }> {
const updates = {};
for (const remoteTable of remoteTables) {
@ -155,37 +173,39 @@ export class RemoteTableService {
const tableName = remoteTable.distantTableName;
if (!distantTable) {
updates[tableName] = [TableUpdate.TABLE_DELETED];
updates[tableName] = [DistantTableUpdate.TABLE_DELETED];
continue;
}
const distantColumnNames = new Set(
distantTable.map((column) => column.columnName),
const distantTableColumnNames = new Set(
distantTable.map((column) =>
getForeignTableColumnName(column.columnName),
),
);
const localColumnNames = new Set(
const foreignTableColumnNames = new Set(
(
await this.fetchTableColumns(workspaceId, remoteTable.localTableName)
).map((column) => column.column_name),
).map((column) => column.columnName),
);
const columnsAdded = [...distantColumnNames].filter(
(columnName) => !localColumnNames.has(columnName),
const columnsAdded = [...distantTableColumnNames].filter(
(columnName) => !foreignTableColumnNames.has(columnName),
);
const columnsDeleted = [...localColumnNames].filter(
(columnName) => !distantColumnNames.has(columnName),
const columnsDeleted = [...foreignTableColumnNames].filter(
(columnName) => !distantTableColumnNames.has(columnName),
);
if (columnsAdded.length > 0) {
updates[tableName] = [
...(updates[tableName] || []),
TableUpdate.COLUMNS_ADDED,
DistantTableUpdate.COLUMNS_ADDED,
];
}
if (columnsDeleted.length > 0) {
updates[tableName] = [
...(updates[tableName] || []),
TableUpdate.COLUMNS_DELETED,
DistantTableUpdate.COLUMNS_DELETED,
];
}
}
@ -431,7 +451,7 @@ export class RemoteTableService {
private async fetchTableColumns(
workspaceId: string,
tableName: string,
): Promise<RemoteTableSchemaColumn[]> {
): Promise<PostgresTableSchemaColumn[]> {
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
workspaceId,
@ -440,11 +460,17 @@ export class RemoteTableService {
const schemaName =
this.workspaceDataSourceService.getSchemaName(workspaceId);
return await workspaceDataSource.query(
`SELECT column_name, data_type
const res = await workspaceDataSource.query(
`SELECT column_name, data_type, udt_name
FROM information_schema.columns
WHERE table_schema = '${schemaName}' AND table_name = '${tableName}'`,
);
return res.map((column) => ({
columnName: column.column_name,
dataType: column.data_type,
udtName: column.udt_name,
}));
}
private async createForeignTable(
@ -452,7 +478,7 @@ export class RemoteTableService {
localTableName: string,
remoteTableInput: RemoteTableInput,
remoteServer: RemoteServerEntity<RemoteServerType>,
distantTableColumns: DistantTableColumn[],
distantTableColumns: PostgresTableSchemaColumn[],
) {
const referencedTable: ReferencedTable = this.buildReferencedTable(
remoteServer,
@ -471,7 +497,7 @@ export class RemoteTableService {
columns: distantTableColumns.map(
(column) =>
({
columnName: camelCase(column.columnName),
columnName: getForeignTableColumnName(column.columnName),
columnType: column.dataType,
distantColumnName: column.columnName,
}) satisfies WorkspaceMigrationForeignColumnDefinition,
@ -500,8 +526,8 @@ export class RemoteTableService {
private async createRemoteTableMetadata(
workspaceId: string,
localTableName: string,
distantTableColumns: DistantTableColumn[],
distantTableIdColumn: DistantTableColumn,
distantTableColumns: PostgresTableSchemaColumn[],
distantTableIdColumn: PostgresTableSchemaColumn,
dataSourceMetadataId: string,
) {
const objectMetadata = await this.objectMetadataService.createOne({

View File

@ -1,4 +0,0 @@
export type RemoteTableSchemaColumn = {
column_name: string;
data_type: string;
};

View File

@ -0,0 +1,5 @@
import camelCase from 'lodash.camelcase';
export const getForeignTableColumnName = (distantTableColumnName: string) => {
return camelCase(distantTableColumnName);
};

View File

@ -0,0 +1,5 @@
export type PostgresTableSchemaColumn = {
columnName: string;
dataType: string;
udtName: string;
};