From e976a1bdfcb13b64e0755cfc70e4a7ea33047828 Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Sat, 27 Apr 2024 11:43:44 +0200 Subject: [PATCH] Uniformize datasources (#5196) ## Context We recently enabled the option to bypass SSL certificate authority validation when establishing a connection to PostgreSQL. Previously, if this validation failed, the server would revert to unencrypted traffic. Now, it maintains encryption even if the SSL certificate check fails. In the process, we overlooked a few DataSource setups, prompting a review of DataSource creation within our code. ## Current State Our DataSource initialization is distributed as follows: - **Database folder**: Contains 'core', 'metadata', and 'raw' DataSources. The 'core' and 'metadata' DataSources manage migrations and static resolver calls to the database. The 'raw' DataSource is utilized in scripts and commands that require handling both aspects. - **typeorm.service.ts script**: These DataSources facilitate multi-schema connections. ## Vision for Discussion - **SystemSchema (formerly core) DataSource**: Manages system schema migrations and system resolvers/repos. The 'core' schema will be renamed to 'system' as the Core API will include parts of the system and workspace schemas. - **MetadataSchema DataSource**: Handles metadata schema migrations and metadata API resolvers/repos. - **(Dynamic) WorkspaceSchema DataSource**: Will be used in the Twenty ORM to access a specific workspace schema. We currently do not support cross-schema joins, so maintaining these DataSources separately should be feasible. Core API resolvers will select the appropriate DataSource based on the field context. - **To be discussed**: The potential need for an AdminDataSource (akin to 'Raw'), which would be used in commands, setup scripts, and the admin panel to connect to any database schema without loading any model. This DataSource should be reserved for cases where utilizing metadata, system, or workspace entities is impractical. ## In This PR - Ensuring all existing DataSources are compliant with the SSL update. - Introducing RawDataSource to eliminate the need for declaring new DataSource() instances in commands. --- packages/twenty-server/scripts/setup-db.ts | 6 ++++-- packages/twenty-server/scripts/truncate-db.ts | 6 ++++-- packages/twenty-server/scripts/utils.ts | 10 ++-------- .../data-seed-demo-workspace.service.ts | 16 ++++------------ .../commands/data-seed-dev-workspace.command.ts | 16 +++++----------- .../database/typeorm/core/core.datasource.ts | 1 + .../src/database/typeorm/raw/raw.datasource.ts | 17 +++++++++++++++++ .../src/database/typeorm/typeorm.service.ts | 5 +++++ 8 files changed, 42 insertions(+), 35 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/raw/raw.datasource.ts diff --git a/packages/twenty-server/scripts/setup-db.ts b/packages/twenty-server/scripts/setup-db.ts index bfdf51133..cefc1245e 100644 --- a/packages/twenty-server/scripts/setup-db.ts +++ b/packages/twenty-server/scripts/setup-db.ts @@ -1,8 +1,10 @@ import console from 'console'; -import { camelToSnakeCase, connectionSource, performQuery } from './utils'; +import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; -connectionSource +import { camelToSnakeCase, performQuery } from './utils'; + +rawDataSource .initialize() .then(async () => { await performQuery( diff --git a/packages/twenty-server/scripts/truncate-db.ts b/packages/twenty-server/scripts/truncate-db.ts index 2e18a3e72..bc5b4feea 100644 --- a/packages/twenty-server/scripts/truncate-db.ts +++ b/packages/twenty-server/scripts/truncate-db.ts @@ -1,10 +1,12 @@ import console from 'console'; -import { connectionSource, performQuery } from './utils'; +import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; + +import { performQuery } from './utils'; async function dropSchemasSequentially() { try { - await connectionSource.initialize(); + await rawDataSource.initialize(); // Fetch all schemas const schemas = await performQuery( diff --git a/packages/twenty-server/scripts/utils.ts b/packages/twenty-server/scripts/utils.ts index 99dbd4b50..5786e7817 100644 --- a/packages/twenty-server/scripts/utils.ts +++ b/packages/twenty-server/scripts/utils.ts @@ -1,12 +1,6 @@ import console from 'console'; -import { DataSource } from 'typeorm'; - -export const connectionSource = new DataSource({ - type: 'postgres', - logging: false, - url: process.env.PG_DATABASE_URL, -}); +import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; export const camelToSnakeCase = (str) => str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`); @@ -18,7 +12,7 @@ export const performQuery = async ( ignoreAlreadyExistsError = false, ) => { try { - const result = await connectionSource.query(query); + const result = await rawDataSource.query(query); withLog && console.log(`Performed '${consoleDescription}' successfully`); diff --git a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts index adc84e677..9be19725a 100644 --- a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts +++ b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts @@ -1,13 +1,12 @@ import { Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; - import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; import { deleteCoreSchema, seedCoreSchema, } from 'src/database/typeorm-seeds/core/demo'; +import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; @Injectable() export class DataSeedDemoWorkspaceService { @@ -18,14 +17,7 @@ export class DataSeedDemoWorkspaceService { async seedDemo(): Promise { try { - const dataSource = new DataSource({ - url: this.environmentService.get('PG_DATABASE_URL'), - type: 'postgres', - logging: true, - schema: 'public', - }); - - await dataSource.initialize(); + await rawDataSource.initialize(); const demoWorkspaceIds = this.environmentService.get('DEMO_WORKSPACE_IDS'); @@ -35,10 +27,10 @@ export class DataSeedDemoWorkspaceService { ); } for (const workspaceId of demoWorkspaceIds) { - await deleteCoreSchema(dataSource, workspaceId); + await deleteCoreSchema(rawDataSource, workspaceId); await this.workspaceManagerService.delete(workspaceId); - await seedCoreSchema(dataSource, workspaceId); + await seedCoreSchema(rawDataSource, workspaceId); await this.workspaceManagerService.initDemo(workspaceId); } } catch (error) { diff --git a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts index e2fa521c3..0f127743b 100644 --- a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts +++ b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts @@ -1,5 +1,5 @@ import { Command, CommandRunner } from 'nest-commander'; -import { DataSource, EntityManager } from 'typeorm'; +import { EntityManager } from 'typeorm'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { seedCompanies } from 'src/database/typeorm-seeds/workspace/companies'; @@ -27,6 +27,7 @@ import { seedCalendarEvents } from 'src/database/typeorm-seeds/workspace/calenda import { seedCalendarChannels } from 'src/database/typeorm-seeds/workspace/calendar-channel'; import { seedCalendarChannelEventAssociations } from 'src/database/typeorm-seeds/workspace/calendar-channel-event-association'; import { seedCalendarEventParticipants } from 'src/database/typeorm-seeds/workspace/calendar-event-participants'; +import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; // TODO: implement dry-run @Command({ @@ -50,19 +51,12 @@ export class DataSeedWorkspaceCommand extends CommandRunner { async run(): Promise { try { - const dataSource = new DataSource({ - url: this.environmentService.get('PG_DATABASE_URL'), - type: 'postgres', - logging: true, - schema: 'core', - }); - for (const workspaceId of this.workspaceIds) { - await dataSource.initialize(); + await rawDataSource.initialize(); - await seedCoreSchema(dataSource, workspaceId); + await seedCoreSchema(rawDataSource, workspaceId); - await dataSource.destroy(); + await rawDataSource.destroy(); const schemaName = await this.workspaceDataSourceService.createWorkspaceDBSchema( diff --git a/packages/twenty-server/src/database/typeorm/core/core.datasource.ts b/packages/twenty-server/src/database/typeorm/core/core.datasource.ts index db0768432..6733370d0 100644 --- a/packages/twenty-server/src/database/typeorm/core/core.datasource.ts +++ b/packages/twenty-server/src/database/typeorm/core/core.datasource.ts @@ -21,6 +21,7 @@ export const typeORMCoreModuleOptions: TypeOrmModuleOptions = { } : undefined, }; + export const connectionSource = new DataSource( typeORMCoreModuleOptions as DataSourceOptions, ); diff --git a/packages/twenty-server/src/database/typeorm/raw/raw.datasource.ts b/packages/twenty-server/src/database/typeorm/raw/raw.datasource.ts new file mode 100644 index 000000000..a50567c43 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/raw/raw.datasource.ts @@ -0,0 +1,17 @@ +import { config } from 'dotenv'; +import { DataSource, DataSourceOptions } from 'typeorm'; +config(); + +const typeORMRawModuleOptions: DataSourceOptions = { + url: process.env.PG_DATABASE_URL, + type: 'postgres', + logging: ['error'], + ssl: + process.env.PG_SSL_ALLOW_SELF_SIGNED === 'true' + ? { + rejectUnauthorized: false, + } + : undefined, +}; + +export const rawDataSource = new DataSource(typeORMRawModuleOptions); diff --git a/packages/twenty-server/src/database/typeorm/typeorm.service.ts b/packages/twenty-server/src/database/typeorm/typeorm.service.ts index c9a05a12a..a3cedb839 100644 --- a/packages/twenty-server/src/database/typeorm/typeorm.service.ts +++ b/packages/twenty-server/src/database/typeorm/typeorm.service.ts @@ -89,6 +89,11 @@ export class TypeORMService implements OnModuleInit, OnModuleDestroy { ? ['query', 'error'] : ['error'], schema, + ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED') + ? { + rejectUnauthorized: false, + } + : undefined, }); await workspaceDataSource.initialize();