From e293abe3320dc6467a47c1dd94b83f7b7bc58402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20M?= Date: Mon, 19 Feb 2024 17:28:40 +0100 Subject: [PATCH] Fix/workspace health type (#4053) * fix: memory issue with truncate command * fix: LINK doesn't have any default value * fix: Cannot convert LINK to column type. * fix: handle old column type and add a warn to fix them manually --- packages/twenty-server/scripts/truncate-db.ts | 56 ++++++++++--------- .../utils/generate-default-value.ts | 5 ++ .../fixer/workspace-type.fixer.ts | 34 ++++++++--- .../services/database-structure.service.ts | 16 +++++- .../workspace-migration-field.factory.ts | 2 +- 5 files changed, 77 insertions(+), 36 deletions(-) diff --git a/packages/twenty-server/scripts/truncate-db.ts b/packages/twenty-server/scripts/truncate-db.ts index 4c616ad98..2e18a3e72 100644 --- a/packages/twenty-server/scripts/truncate-db.ts +++ b/packages/twenty-server/scripts/truncate-db.ts @@ -2,31 +2,35 @@ import console from 'console'; import { connectionSource, performQuery } from './utils'; -connectionSource - .initialize() - .then(async () => { - await performQuery( +async function dropSchemasSequentially() { + try { + await connectionSource.initialize(); + + // Fetch all schemas + const schemas = await performQuery( ` - CREATE OR REPLACE FUNCTION drop_all() RETURNS VOID AS $$ - DECLARE schema_item RECORD; - BEGIN - FOR schema_item IN - SELECT subrequest."name" as schema_name - FROM (SELECT n.nspname AS "name" - FROM pg_catalog.pg_namespace n - WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema') as subrequest - LOOP - EXECUTE 'DROP SCHEMA ' || schema_item.schema_name || ' CASCADE'; - END LOOP; - RETURN; - END; - $$ LANGUAGE plpgsql; - - SELECT drop_all (); - `, - 'Dropping all schemas...', + SELECT n.nspname AS "schema_name" + FROM pg_catalog.pg_namespace n + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + `, + 'Fetching schemas...', ); - }) - .catch((err) => { - console.error('Error during Data Source initialization:', err); - }); + + // Iterate over each schema and drop it + // This is to avoid dropping all schemas at once, which would cause an out of shared memory error + for (const schema of schemas) { + await performQuery( + ` + DROP SCHEMA IF EXISTS "${schema.schema_name}" CASCADE; + `, + `Dropping schema ${schema.schema_name}...`, + ); + } + + console.log('All schemas dropped successfully.'); + } catch (err) { + console.error('Error during schema dropping:', err); + } +} + +dropSchemasSequentially(); diff --git a/packages/twenty-server/src/metadata/field-metadata/utils/generate-default-value.ts b/packages/twenty-server/src/metadata/field-metadata/utils/generate-default-value.ts index 83cb7a612..a6038042f 100644 --- a/packages/twenty-server/src/metadata/field-metadata/utils/generate-default-value.ts +++ b/packages/twenty-server/src/metadata/field-metadata/utils/generate-default-value.ts @@ -17,6 +17,11 @@ export function generateDefaultValue( firstName: '', lastName: '', }; + case FieldMetadataType.LINK: + return { + url: '', + label: '', + }; default: return null; } diff --git a/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-type.fixer.ts b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-type.fixer.ts index 3392cb046..4125c9477 100644 --- a/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-type.fixer.ts +++ b/packages/twenty-server/src/workspace/workspace-health/fixer/workspace-type.fixer.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { EntityManager } from 'typeorm'; @@ -10,13 +10,20 @@ import { WorkspaceMigrationBuilderAction } from 'src/workspace/workspace-migrati import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; import { WorkspaceMigrationEntity } from 'src/metadata/workspace-migration/workspace-migration.entity'; -import { WorkspaceMigrationFieldFactory } from 'src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory'; +import { + FieldMetadataUpdate, + WorkspaceMigrationFieldFactory, +} from 'src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory'; import { DatabaseStructureService } from 'src/workspace/workspace-health/services/database-structure.service'; import { AbstractWorkspaceFixer } from './abstract-workspace.fixer'; +const oldDataTypes = ['integer']; + @Injectable() export class WorkspaceTypeFixer extends AbstractWorkspaceFixer { + private readonly logger = new Logger(WorkspaceTypeFixer.name); + constructor( private readonly workspaceMigrationFieldFactory: WorkspaceMigrationFieldFactory, private readonly databaseStructureService: DatabaseStructureService, @@ -40,28 +47,39 @@ export class WorkspaceTypeFixer extends AbstractWorkspaceFixer[], ): Promise[]> { - const fieldMetadataUpdateCollection = issues.map((issue) => { - if (!issue.columnStructure?.dataType) { + const fieldMetadataUpdateCollection: FieldMetadataUpdate[] = []; + + for (const issue of issues) { + const dataType = issue.columnStructure?.dataType; + + if (!dataType) { throw new Error('Column structure data type is missing'); } const type = this.databaseStructureService.getFieldMetadataTypeFromPostgresDataType( - issue.columnStructure?.dataType, + dataType, ); + if (oldDataTypes.includes(dataType)) { + this.logger.warn( + `Old data type detected for column ${issue.columnStructure?.columnName} with data type ${dataType}. Please update the column data type manually.`, + ); + continue; + } + if (!type) { throw new Error("Can't find field metadata type from column structure"); } - return { + fieldMetadataUpdateCollection.push({ current: { ...issue.fieldMetadata, type, }, altered: issue.fieldMetadata, - }; - }); + }); + } return this.workspaceMigrationFieldFactory.create( objectMetadataCollection, diff --git a/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts b/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts index 60803726c..1afe299fa 100644 --- a/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts +++ b/packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts @@ -16,6 +16,8 @@ import { } from 'src/metadata/field-metadata/field-metadata.entity'; import { fieldMetadataTypeToColumnType } from 'src/metadata/workspace-migration/utils/field-metadata-type-to-column-type.util'; import { serializeTypeDefaultValue } from 'src/metadata/field-metadata/utils/serialize-type-default-value.util'; +import { isCompositeFieldMetadataType } from 'src/metadata/field-metadata/utils/is-composite-field-metadata-type.util'; +import { isRelationFieldMetadataType } from 'src/workspace/utils/is-relation-field-metadata-type.util'; @Injectable() export class DatabaseStructureService { @@ -165,8 +167,20 @@ export class DatabaseStructureService { postgresDataType: string, ): FieldMetadataType | null { const mainDataSource = this.typeORMService.getMainDataSource(); + const types = Object.values(FieldMetadataType).filter((type) => { + // We're skipping composite and relation types, as they're not directly mapped to a column type + if (isCompositeFieldMetadataType(type)) { + return false; + } - for (const type in FieldMetadataType) { + if (isRelationFieldMetadataType(type)) { + return false; + } + + return true; + }); + + for (const type of types) { const typeORMType = fieldMetadataTypeToColumnType( FieldMetadataType[type], ) as ColumnType; diff --git a/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts b/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts index aedb33f47..9337b6c14 100644 --- a/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts +++ b/packages/twenty-server/src/workspace/workspace-migration-builder/factories/workspace-migration-field.factory.ts @@ -16,7 +16,7 @@ import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-tar import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory'; import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util'; -interface FieldMetadataUpdate { +export interface FieldMetadataUpdate { current: FieldMetadataEntity; altered: FieldMetadataEntity; }