Remove usages of connectToDataSource and use workspaceDataSource (#11873)

In this PR we are

1. cleaning typeORM service by removing connectToDataSource method
2. using workspaceDataSource instead of mainDataSource when possible,
and replacing raw SQL with workspaceRepository methods to use

packages/twenty-server/src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner.ts

@@ -1,7 +1,7 @@
 import chalk from 'chalk';
 import { Option } from 'nest-commander';
-import { In, MoreThanOrEqual, Repository } from 'typeorm';
 import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
+import { In, MoreThanOrEqual, Repository } from 'typeorm';
 
 import { MigrationCommandRunner } from 'src/database/commands/command-runners/migration.command-runner';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
@@ -138,10 +138,10 @@ export abstract class ActiveOrSuspendedWorkspacesMigrationCommandRunner<
 
       try {
         const dataSource =
-          await this.twentyORMGlobalManager.getDataSourceForWorkspace(
+          await this.twentyORMGlobalManager.getDataSourceForWorkspace({
             workspaceId,
-            false,
-          );
+            shouldFailIfMetadataNotFound: false,
+          });
 
         await this.runOnWorkspace({
           options,

packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts

@@ -46,7 +46,7 @@ export class DataSeedDemoWorkspaceService {
         const appVersion = this.twentyConfigService.get('APP_VERSION');
 
         await seedCoreSchema({
-          workspaceDataSource: rawDataSource,
+          dataSource: rawDataSource,
           workspaceId,
           appVersion,
           seedBilling: false,

packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts

@@ -30,9 +30,6 @@ import { seedPeople } from 'src/database/typeorm-seeds/workspace/seedPeople';
 import { seedWorkspaceMember } from 'src/database/typeorm-seeds/workspace/workspace-members';
 import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource';
 import { TypeORMService } from 'src/database/typeorm/typeorm.service';
-import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
-import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
-import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
 import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
 import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@@ -45,12 +42,12 @@ import { SURVEY_RESULTS_METADATA_SEEDS } from 'src/engine/seeder/metadata-seeds/
 import { SeederService } from 'src/engine/seeder/seeder.service';
 import { WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
 import { shouldSeedWorkspaceFavorite } from 'src/engine/utils/should-seed-workspace-favorite';
+import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
 import { createWorkspaceViews } from 'src/engine/workspace-manager/standard-objects-prefill-data/create-workspace-views';
 import { seedViewWithDemoData } from 'src/engine/workspace-manager/standard-objects-prefill-data/seed-view-with-demo-data';
 import { opportunitiesTableByStageView } from 'src/engine/workspace-manager/standard-objects-prefill-data/views/opportunity-table-by-stage.view';
 import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
 import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
-
 // TODO: implement dry-run
 @Command({
   name: 'workspace:seed:dev',
@@ -66,11 +63,10 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
     private readonly typeORMService: TypeORMService,
     private readonly fieldMetadataService: FieldMetadataService,
     private readonly objectMetadataService: ObjectMetadataService,
-    @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace)
-    private readonly workspaceSchemaCache: CacheStorageService,
     private readonly seederService: SeederService,
     private readonly workspaceManagerService: WorkspaceManagerService,
     private readonly twentyConfigService: TwentyConfigService,
+    private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
   ) {
     super();
   }
@@ -92,7 +88,13 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
   }
 
   async createWorkspaceSchema(workspaceId: string) {
-    await this.workspaceSchemaCache.flush();
+    const workspaceCachedMetadataVersion =
+      await this.workspaceCacheStorageService.getMetadataVersion(workspaceId);
+
+    await this.workspaceCacheStorageService.flush(
+      workspaceId,
+      workspaceCachedMetadataVersion,
+    );
 
     await rawDataSource.initialize();
 
@@ -100,7 +102,7 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
     const appVersion = this.twentyConfigService.get('APP_VERSION');
 
     await seedCoreSchema({
-      workspaceDataSource: rawDataSource,
+      dataSource: rawDataSource,
       workspaceId,
       seedBilling: isBillingEnabled,
       appVersion,
@@ -117,10 +119,9 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
         workspaceId,
       );
 
-    const workspaceDataSource =
-      await this.typeORMService.connectToDataSource(dataSourceMetadata);
+    const mainDataSource = this.typeORMService.getMainDataSource();
 
-    if (!workspaceDataSource) {
+    if (!mainDataSource) {
       throw new Error('Could not connect to workspace data source');
     }
 
@@ -140,10 +141,7 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
         workspaceId,
       );
 
-      await this.seedStandardObjectRecords(
-        workspaceDataSource,
-        dataSourceMetadata,
-      );
+      await this.seedStandardObjectRecords(mainDataSource, dataSourceMetadata);
 
       await this.seederService.seedCustomObjects(
         dataSourceMetadata.id,
@@ -161,15 +159,13 @@ export class DataSeedWorkspaceCommand extends CommandRunner {
     } catch (error) {
       this.logger.error(error);
     }
-
-    await this.typeORMService.disconnectFromDataSource(dataSourceMetadata.id);
   }
 
   async seedStandardObjectRecords(
-    workspaceDataSource: DataSource,
+    mainDataSource: DataSource,
     dataSourceMetadata: DataSourceEntity,
   ) {
-    await workspaceDataSource.transaction(
+    await mainDataSource.transaction(
       async (entityManager: WorkspaceEntityManager) => {
         const { objectMetadataStandardIdToIdMap } =
           await this.objectMetadataService.getObjectMetadataStandardIdToIdMap(

packages/twenty-server/src/database/commands/database-command.module.ts

@@ -8,6 +8,7 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s
 import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
 import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
 import { SeederModule } from 'src/engine/seeder/seeder.module';
+import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
 import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
 
 @Module({
@@ -21,6 +22,7 @@ import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-m
     SeederModule,
     WorkspaceManagerModule,
     DataSourceModule,
+    WorkspaceCacheStorageModule,
   ],
   providers: [DataSeedWorkspaceCommand, ConfirmationQuestion],
 })

packages/twenty-server/src/database/commands/upgrade-version-command/0-43/0-43-migrate-rich-text-content-patch.command.ts

@@ -227,12 +227,11 @@ export class MigrateRichTextContentPatchCommand extends ActiveOrSuspendedWorkspa
       const schemaName =
         this.workspaceDataSourceService.getSchemaName(workspaceId);
 
-      const failOnMetadataCacheMiss = false;
       const workspaceDataSource =
-        await this.twentyORMGlobalManager.getDataSourceForWorkspace(
+        await this.twentyORMGlobalManager.getDataSourceForWorkspace({
           workspaceId,
-          failOnMetadataCacheMiss,
-        );
+          shouldFailIfMetadataNotFound: false,
+        });
 
       const rows = await workspaceDataSource.query(
         `SELECT id, "${richTextField.name}" FROM "${schemaName}"."${computeTableName(objectMetadata.nameSingular, objectMetadata.isCustom)}" WHERE "${richTextField.name}" IS NOT NULL`,

packages/twenty-server/src/database/commands/upgrade-version-command/0-51/0-51-update-workflow-trigger-type-enum.command.ts

@@ -1,18 +1,18 @@
 import { InjectRepository } from '@nestjs/typeorm';
 
 import { Command } from 'nest-commander';
-import { Repository } from 'typeorm';
 import { isDefined } from 'twenty-shared/utils';
+import { Repository } from 'typeorm';
 
 import {
   ActiveOrSuspendedWorkspacesMigrationCommandRunner,
   RunOnWorkspaceArgs,
 } from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
-import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
 import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
 import { computeTableName } from 'src/engine/utils/compute-table-name.util';
-import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
 
 @Command({
   name: 'upgrade:0-51:upgrade-created-by-enum',
@@ -43,7 +43,9 @@ export class UpgradeCreatedByEnumCommand extends ActiveOrSuspendedWorkspacesMigr
       this.workspaceDataSourceService.getSchemaName(workspaceId);
 
     const workspaceDataSource =
-      await this.twentyORMGlobalManager.getDataSourceForWorkspace(workspaceId);
+      await this.twentyORMGlobalManager.getDataSourceForWorkspace({
+        workspaceId,
+      });
 
     const objectMetadatas = await this.objectMetadataRepository.find({
       where: {

packages/twenty-server/src/database/typeorm-seeds/core/index.ts

@@ -7,7 +7,7 @@ import { seedUsers } from 'src/database/typeorm-seeds/core/users';
 import { seedWorkspaces } from 'src/database/typeorm-seeds/core/workspaces';
 
 type SeedCoreSchemaArgs = {
-  workspaceDataSource: DataSource;
+  dataSource: DataSource;
   workspaceId: string;
   appVersion: string | undefined;
   seedBilling?: boolean;
@@ -16,7 +16,7 @@ type SeedCoreSchemaArgs = {
 
 export const seedCoreSchema = async ({
   appVersion,
-  workspaceDataSource,
+  dataSource,
   workspaceId,
   seedBilling = true,
   seedFeatureFlags: shouldSeedFeatureFlags = true,
@@ -24,23 +24,19 @@ export const seedCoreSchema = async ({
   const schemaName = 'core';
 
   await seedWorkspaces({
-    workspaceDataSource,
+    dataSource,
     schemaName,
     workspaceId,
     appVersion,
   });
-  await seedUsers(workspaceDataSource, schemaName);
-  await seedUserWorkspaces(workspaceDataSource, schemaName, workspaceId);
+  await seedUsers(dataSource, schemaName);
+  await seedUserWorkspaces(dataSource, schemaName, workspaceId);
 
   if (shouldSeedFeatureFlags) {
-    await seedFeatureFlags(workspaceDataSource, schemaName, workspaceId);
+    await seedFeatureFlags(dataSource, schemaName, workspaceId);
   }
 
   if (seedBilling) {
-    await seedBillingSubscriptions(
-      workspaceDataSource,
-      schemaName,
-      workspaceId,
-    );
+    await seedBillingSubscriptions(dataSource, schemaName, workspaceId);
   }
 };

packages/twenty-server/src/database/typeorm-seeds/core/workspaces.ts

@@ -10,7 +10,7 @@ export const SEED_APPLE_WORKSPACE_ID = '20202020-1c25-4d02-bf25-6aeccf7ea419';
 export const SEED_ACME_WORKSPACE_ID = '3b8e6458-5fc1-4e63-8563-008ccddaa6db';
 
 export type SeedWorkspaceArgs = {
-  workspaceDataSource: DataSource;
+  dataSource: DataSource;
   schemaName: string;
   workspaceId: string;
   appVersion: string | undefined;
@@ -33,7 +33,7 @@ type WorkspaceSeederFields = Pick<
 
 export const seedWorkspaces = async ({
   schemaName,
-  workspaceDataSource,
+  dataSource,
   workspaceId,
   appVersion,
 }: SeedWorkspaceArgs) => {
@@ -60,7 +60,7 @@ export const seedWorkspaces = async ({
     },
   };
 
-  await workspaceDataSource
+  await dataSource
     .createQueryBuilder()
     .insert()
     .into(`${schemaName}.${tableName}`, workspaceSeederFields)
@@ -70,11 +70,11 @@ export const seedWorkspaces = async ({
 };
 
 export const deleteWorkspaces = async (
-  workspaceDataSource: DataSource,
+  dataSource: DataSource,
   schemaName: string,
   workspaceId: string,
 ) => {
-  await workspaceDataSource
+  await dataSource
     .createQueryBuilder()
     .delete()
     .from(`${schemaName}.${tableName}`)

packages/twenty-server/src/database/typeorm/typeorm.service.ts

@@ -22,12 +22,9 @@ import { TwoFactorMethod } from 'src/engine/core-modules/two-factor-method/two-f
 import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
 import { User } from 'src/engine/core-modules/user/user.entity';
 import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
-import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
 @Injectable()
 export class TypeORMService implements OnModuleInit, OnModuleDestroy {
   private mainDataSource: DataSource;
-  private dataSources: Map<string, DataSource> = new Map();
-  private isDatasourceInitializing: Map<string, boolean> = new Map();
 
   constructor(private readonly twentyConfigService: TwentyConfigService) {
     this.mainDataSource = new DataSource({
@@ -73,75 +70,6 @@ export class TypeORMService implements OnModuleInit, OnModuleDestroy {
     return this.mainDataSource;
   }
 
-  public async connectToDataSource(
-    dataSource: DataSourceEntity,
-  ): Promise<DataSource | undefined> {
-    const isMultiDatasourceEnabled = false;
-
-    if (isMultiDatasourceEnabled) {
-      // Wait for a bit before trying again if another initialization is in progress
-      while (this.isDatasourceInitializing.get(dataSource.id)) {
-        await new Promise((resolve) => setTimeout(resolve, 10));
-      }
-
-      if (this.dataSources.has(dataSource.id)) {
-        return this.dataSources.get(dataSource.id);
-      }
-
-      this.isDatasourceInitializing.set(dataSource.id, true);
-
-      try {
-        const dataSourceInstance =
-          await this.createAndInitializeDataSource(dataSource);
-
-        this.dataSources.set(dataSource.id, dataSourceInstance);
-
-        return dataSourceInstance;
-      } finally {
-        this.isDatasourceInitializing.delete(dataSource.id);
-      }
-    }
-
-    return this.mainDataSource;
-  }
-
-  private async createAndInitializeDataSource(
-    dataSource: DataSourceEntity,
-  ): Promise<DataSource> {
-    const schema = dataSource.schema;
-
-    const workspaceDataSource = new DataSource({
-      url: dataSource.url ?? this.twentyConfigService.get('PG_DATABASE_URL'),
-      type: 'postgres',
-      logging:
-        this.twentyConfigService.get('NODE_ENV') === NodeEnvironment.development
-          ? ['query', 'error']
-          : ['error'],
-      schema,
-      ssl: this.twentyConfigService.get('PG_SSL_ALLOW_SELF_SIGNED')
-        ? {
-            rejectUnauthorized: false,
-          }
-        : undefined,
-    });
-
-    await workspaceDataSource.initialize();
-
-    return workspaceDataSource;
-  }
-
-  public async disconnectFromDataSource(dataSourceId: string) {
-    if (!this.dataSources.has(dataSourceId)) {
-      return;
-    }
-
-    const dataSource = this.dataSources.get(dataSourceId);
-
-    await dataSource?.destroy();
-
-    this.dataSources.delete(dataSourceId);
-  }
-
   public async createSchema(schemaName: string): Promise<string> {
     const queryRunner = this.mainDataSource.createQueryRunner();
 
@@ -168,10 +96,5 @@ export class TypeORMService implements OnModuleInit, OnModuleDestroy {
   async onModuleDestroy() {
     // Destroy main data source "default" schema
     await this.mainDataSource.destroy();
-
-    // Destroy all workspace data sources
-    for (const [, dataSource] of this.dataSources) {
-      await dataSource.destroy();
-    }
   }
 }