multi tenant schemas poc (#1569)

* Multi-tenant db schemas POC

* fix tests and use query builders

* remove synchronize

* restore updatedAt

* remove unnecessary import

* use queryRunner

* fix camelcase

* add migrations for standard objects

* Multi-tenant db schemas POC

* fix tests and use query builders

* remove synchronize

* restore updatedAt

* remove unnecessary import

* use queryRunner

* fix camelcase

* add migrations for standard objects

* add metadata

* add comments

* remove migrations for now

* do not allow connection to public schema for non-remote workspace connection

* rename getLastDataSourceMetadataFromWorkspaceIdOrFail

* remove schema creation

* remove module import

server/src/core/core.module.ts

@@ -14,6 +14,7 @@ import { AttachmentModule } from './attachment/attachment.module';
 import { ActivityModule } from './activity/activity.module';
 import { ViewModule } from './view/view.module';
 import { FavoriteModule } from './favorite/favorite.module';
+import { TenantModule } from './tenant/tenant.module';
 
 @Module({
   imports: [
@@ -31,6 +32,7 @@ import { FavoriteModule } from './favorite/favorite.module';
     ActivityModule,
     ViewModule,
     FavoriteModule,
+    TenantModule,
   ],
   exports: [
     AuthModule,

server/src/core/tenant/datasource/datasource.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+
+import { DataSourceService } from './services/datasource.service';
+
+@Module({
+  exports: [DataSourceService],
+  providers: [DataSourceService],
+})
+export class DataSourceModule {}

server/src/core/tenant/datasource/entities/base.entity.ts

@@ -0,0 +1,7 @@
+export const baseColumns = {
+  id: {
+    primary: true,
+    type: 'uuid',
+    generated: 'uuid',
+  },
+};

server/src/core/tenant/datasource/entities/data-source-metadata.entity.ts

@@ -0,0 +1,73 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm';
+
+// export const dataSourceEntity = new EntitySchema({
+//   name: 'data_sources',
+//   columns: {
+//     id: {
+//       primary: true,
+//       type: 'uuid',
+//       generated: 'uuid',
+//     },
+//     url: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     schema: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     type: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     name: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     is_remote: {
+//       type: 'boolean',
+//       nullable: true,
+//       default: false,
+//     },
+//     workspace_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//   },
+// });
+
+@Entity('data_source_metadata')
+export class DataSourceMetadata {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @CreateDateColumn()
+  created_at: Date;
+
+  @UpdateDateColumn()
+  updated_at: Date;
+
+  @Column({ nullable: true })
+  url: string;
+
+  @Column({ nullable: true })
+  schema: string;
+
+  @Column({ default: 'postgres' })
+  type: string;
+
+  @Column({ nullable: true })
+  name: string;
+
+  @Column({ default: false })
+  is_remote: boolean;
+
+  @Column({ nullable: false })
+  workspace_id: string;
+}

server/src/core/tenant/datasource/entities/field-metadata.entity.ts

@@ -0,0 +1,85 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm';
+
+import { ObjectMetadata } from './object-metadata';
+
+// export const fieldMetadataEntity = new EntitySchema({
+//   name: 'field_metadata',
+//   columns: {
+//     id: {
+//       primary: true,
+//       type: 'uuid',
+//       generated: 'uuid',
+//     },
+//     object_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//     type: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     name: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     is_custom: {
+//       type: 'boolean',
+//       nullable: true,
+//       default: false,
+//     },
+//     workspace_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//   },
+//   relations: {
+//     object: {
+//       type: 'many-to-one',
+//       target: 'object_metadata',
+//       joinColumn: {
+//         name: 'object_id',
+//         referencedColumnName: 'id',
+//       },
+//       inverseSide: 'fields',
+//     },
+//   } as any,
+// });
+
+@Entity('field_metadata')
+export class FieldMetadata {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @CreateDateColumn()
+  created_at: Date;
+
+  @UpdateDateColumn()
+  updated_at: Date;
+
+  @Column({ nullable: false })
+  object_id: string;
+
+  @Column({ nullable: false })
+  type: string;
+
+  @Column({ nullable: false })
+  name: string;
+
+  @Column({ default: false })
+  is_custom: boolean;
+
+  @Column({ nullable: false })
+  workspace_id: string;
+
+  @ManyToOne(() => ObjectMetadata, (object) => object.fields)
+  @JoinColumn({ name: 'object_id' })
+  object: ObjectMetadata;
+}

server/src/core/tenant/datasource/entities/object-metadata.ts

@@ -0,0 +1,72 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  OneToMany,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm';
+
+import { FieldMetadata } from './field-metadata.entity';
+
+// export const objectMetadataEntity = new EntitySchema({
+//   name: 'object_metadata',
+//   columns: {
+//     id: {
+//       primary: true,
+//       type: 'uuid',
+//       generated: 'uuid',
+//     },
+//     data_source_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//     name: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     is_custom: {
+//       type: 'boolean',
+//       nullable: true,
+//       default: false,
+//     },
+//     workspace_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//   },
+//   relations: {
+//     fields: {
+//       type: 'one-to-many',
+//       target: 'field_metadata',
+//       inverseSide: 'object',
+//     },
+//   } as any,
+// });
+
+@Entity('object_metadata')
+export class ObjectMetadata {
+  @PrimaryGeneratedColumn('uuid')
+  id: string;
+
+  @CreateDateColumn()
+  created_at: Date;
+
+  @UpdateDateColumn()
+  updated_at: Date;
+
+  @Column({ nullable: false })
+  data_source_id: string;
+
+  @Column({ nullable: false })
+  name: string;
+
+  @Column({ default: false })
+  is_custom: boolean;
+
+  @Column({ nullable: false })
+  workspace_id: string;
+
+  @OneToMany(() => FieldMetadata, (field) => field.object)
+  fields: FieldMetadata[];
+}

server/src/core/tenant/datasource/entities/relation-medata.ts

@@ -0,0 +1,30 @@
+// export const relationMetadataEntity = new EntitySchema({
+//   name: 'relation_metadata',
+//   columns: {
+//     id: {
+//       primary: true,
+//       type: 'uuid',
+//       generated: 'uuid',
+//     },
+//     source_field_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//     target_object_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//     target_foreign_key: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//     type: {
+//       type: 'text',
+//       nullable: true,
+//     },
+//     workspace_id: {
+//       type: 'uuid',
+//       nullable: true,
+//     },
+//   },
+// });

server/src/core/tenant/datasource/services/datasource.service.ts

@@ -0,0 +1,210 @@
+import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+
+import { DataSource, EntitySchema } from 'typeorm';
+import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions';
+
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+import { DataSourceMetadata } from 'src/core/tenant/datasource/entities/data-source-metadata.entity';
+import { ObjectMetadata } from 'src/core/tenant/datasource/entities/object-metadata';
+import { FieldMetadata } from 'src/core/tenant/datasource/entities/field-metadata.entity';
+import { baseColumns } from 'src/core/tenant/datasource/entities/base.entity';
+import {
+  convertFieldTypeToPostgresType,
+  sanitizeColumnName,
+  uuidToBase36,
+} from 'src/core/tenant/datasource/utils/datasource.util';
+
+@Injectable()
+export class DataSourceService implements OnModuleInit, OnModuleDestroy {
+  private mainDataSource: DataSource;
+  private metadataDataSource: DataSource;
+  private connectionOptions: PostgresConnectionOptions;
+  private dataSources = new Map<string, DataSource>();
+
+  constructor(environmentService: EnvironmentService) {
+    this.connectionOptions = {
+      url: environmentService.getPGDatabaseUrl(),
+      type: 'postgres',
+      logging: false,
+      schema: 'public',
+    };
+    this.mainDataSource = new DataSource(this.connectionOptions);
+    this.metadataDataSource = new DataSource({
+      ...this.connectionOptions,
+      schema: 'metadata',
+      synchronize: true, // TODO: remove this in production
+      entities: [DataSourceMetadata, ObjectMetadata, FieldMetadata],
+    });
+  }
+
+  /**
+   *
+   * Returns the schema name for a given workspaceId
+   * @param workspaceId
+   * @returns string
+   */
+  public getSchemaName(workspaceId: string): string {
+    return `workspace_${uuidToBase36(workspaceId)}`;
+  }
+
+  /**
+   * Creates a new schema for a given workspaceId
+   * @param workspaceId
+   * @returns Promise<void>
+   */
+  public async createWorkspaceSchema(workspaceId: string): Promise<void> {
+    const schemaName = this.getSchemaName(workspaceId);
+
+    const queryRunner = this.mainDataSource.createQueryRunner();
+    await queryRunner.createSchema(schemaName, true);
+    await queryRunner.release();
+
+    await this.insertNewDataSourceMetadata(workspaceId, schemaName);
+  }
+
+  /**
+   * Inserts a new data source
+   * @param workspaceId
+   * @param workspaceSchema this can be computed from the workspaceId but it won't be the case for remote data sources
+   * @returns Promise<void>
+   */
+  private async insertNewDataSourceMetadata(
+    workspaceId: string,
+    workspaceSchema: string,
+  ): Promise<void> {
+    await this.metadataDataSource
+      ?.createQueryBuilder()
+      .insert()
+      .into(DataSourceMetadata)
+      .values({
+        workspace_id: workspaceId,
+        schema: workspaceSchema,
+      })
+      .execute();
+  }
+
+  /**
+   * Connects to a workspace data source using the workspace metadata. Returns a cached connection if it exists.
+   * @param workspaceId
+   * @returns Promise<DataSource | undefined>
+   */
+  public async connectToWorkspaceDataSource(
+    workspaceId: string,
+  ): Promise<DataSource | undefined> {
+    // if (this.dataSources.has(workspaceId)) {
+    //   const cachedDataSource = this.dataSources.get(workspaceId);
+    //   return cachedDataSource;
+    // }
+
+    const dataSourceMetadata =
+      await this.getLastDataSourceMetadataFromWorkspaceIdOrFail(workspaceId);
+
+    const schema = dataSourceMetadata.schema;
+
+    // Probably not needed as we will ask for the schema name OR store public by default if it's remote
+    if (!schema && !dataSourceMetadata.is_remote) {
+      throw Error(
+        "No schema found for this non-remote data source, we don't want to fallback to public for workspace data sources.",
+      );
+    }
+
+    const metadata = await this.fetchObjectsAndFieldsFromMetadata(workspaceId);
+
+    const entities = this.convertMetadataToEntities(metadata);
+
+    const workspaceDataSource = new DataSource({
+      ...this.connectionOptions,
+      schema,
+      entities: entities,
+      synchronize: true, // TODO: remove this in production
+    });
+
+    await workspaceDataSource.initialize();
+
+    return workspaceDataSource;
+    // this.dataSources.set(workspaceId, workspaceDataSource);
+
+    // return this.dataSources.get(workspaceId);
+  }
+
+  /**
+   * Returns the last data source metadata for a given workspaceId
+   * In the future we should handle multiple data sources.
+   * Most likely the twenty workspace connection and n remote connections should be fetched.
+   *
+   * @param workspaceId
+   * @returns Promise<DataSourceMetadata>
+   */
+  private async getLastDataSourceMetadataFromWorkspaceIdOrFail(
+    workspaceId: string,
+  ): Promise<DataSourceMetadata> {
+    return this.metadataDataSource
+      ?.createQueryBuilder()
+      .select('data_source')
+      .from(DataSourceMetadata, 'data_source')
+      .where('data_source.workspace_id = :workspaceId', { workspaceId })
+      .orderBy('data_source.created_at', 'DESC')
+      .getOneOrFail();
+  }
+
+  /**
+   * Returns all the objects and fields for a given workspaceId and the first associated data source metadata registered.
+   *
+   * @param workspaceId
+   * @returns
+   */
+  public async fetchObjectsAndFieldsFromMetadata(workspaceId: string) {
+    const dataSource =
+      await this.getLastDataSourceMetadataFromWorkspaceIdOrFail(workspaceId);
+
+    const objectRepository =
+      this.metadataDataSource.getRepository(ObjectMetadata);
+
+    return await objectRepository
+      .createQueryBuilder('object_metadata')
+      .select(['object_metadata.id', 'object_metadata.name'])
+      .leftJoinAndSelect('object_metadata.fields', 'field')
+      .where('object_metadata.data_source_id = :dataSourceId', {
+        dataSourceId: dataSource?.id,
+      })
+      .getMany();
+  }
+
+  /**
+   * Converts the metadata to entities that can be interpreted by typeorm.
+   * @param metadata
+   * @returns EntitySchema[]
+   *
+   */
+  public convertMetadataToEntities(metadata): EntitySchema[] {
+    const entities = metadata.map((object) => {
+      return new EntitySchema({
+        name: object.name,
+        columns: {
+          ...baseColumns,
+          ...object.fields.reduce((columns, field) => {
+            return {
+              ...columns,
+              [sanitizeColumnName(field.name)]: {
+                type: convertFieldTypeToPostgresType(field.type),
+                nullable: true,
+              },
+            };
+          }, {}),
+        },
+      });
+    });
+
+    return entities;
+  }
+
+  async onModuleInit() {
+    await this.mainDataSource.initialize();
+    await this.metadataDataSource.initialize();
+  }
+
+  async onModuleDestroy(): Promise<void> {
+    await this.mainDataSource.destroy();
+    await this.metadataDataSource.destroy();
+  }
+}

server/src/core/tenant/datasource/utils/datasource.util.ts

@@ -0,0 +1,47 @@
+/**
+ * Converts a UUID to a base 36 string.
+ * This is used to generate the schema name since hyphens from workspace uuid are not allowed in postgres schema names.
+ *
+ * @param uuid
+ * @returns
+ */
+export function uuidToBase36(uuid: string): string {
+  const hexString = uuid.replace(/-/g, '');
+  const base10Number = BigInt('0x' + hexString);
+  const base36String = base10Number.toString(36);
+  return base36String;
+}
+
+/**
+ * Sanitizes a column name by replacing all non-alphanumeric characters with an underscore.
+ * Note: Probablay not the best way to do this, leaving it here as a placeholder for now.
+ *
+ * @param columnName
+ * @returns string
+ */
+export function sanitizeColumnName(columnName: string): string {
+  return columnName.replace(/[^a-zA-Z0-9]/g, '_');
+}
+
+/**
+ * Converts a field type to a postgres type. Field types are defined in the UI.
+ *
+ * @param fieldType
+ * @returns string
+ */
+export function convertFieldTypeToPostgresType(fieldType: string): string {
+  switch (fieldType) {
+    case 'text':
+      return 'text';
+    case 'url':
+      return 'text';
+    case 'number':
+      return 'numeric';
+    case 'boolean':
+      return 'boolean';
+    case 'date':
+      return 'timestamp';
+    default:
+      return 'text';
+  }
+}

server/src/core/tenant/metadata/metadata.controller.ts

@@ -0,0 +1,19 @@
+import { Controller, Get, UseGuards } from '@nestjs/common';
+
+import { Workspace } from '@prisma/client';
+
+import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
+import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
+
+import { MetadataService } from './metadata.service';
+
+@UseGuards(JwtAuthGuard)
+@Controller('metadata')
+export class MetadataController {
+  constructor(private readonly metadataService: MetadataService) {}
+
+  @Get()
+  async getMetadata(@AuthWorkspace() workspace: Workspace) {
+    return this.metadataService.fetchMetadataFromWorkspaceId(workspace.id);
+  }
+}

server/src/core/tenant/metadata/metadata.module.ts

@@ -0,0 +1,13 @@
+import { Module } from '@nestjs/common';
+
+import { DataSourceModule } from 'src/core/tenant/datasource/datasource.module';
+
+import { MetadataController } from './metadata.controller';
+import { MetadataService } from './metadata.service';
+
+@Module({
+  imports: [DataSourceModule],
+  controllers: [MetadataController],
+  providers: [MetadataService],
+})
+export class MetadataModule {}

server/src/core/tenant/metadata/metadata.service.ts

@@ -0,0 +1,16 @@
+import { Injectable } from '@nestjs/common';
+
+import { DataSourceService } from 'src/core/tenant/datasource/services/datasource.service';
+
+@Injectable()
+export class MetadataService {
+  constructor(private readonly dataSourceService: DataSourceService) {}
+
+  public async fetchMetadataFromWorkspaceId(workspaceId: string) {
+    await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
+
+    return await this.dataSourceService.fetchObjectsAndFieldsFromMetadata(
+      workspaceId,
+    );
+  }
+}

server/src/core/tenant/tenant.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+
+import { DataSourceModule } from './datasource/datasource.module';
+import { MetadataModule } from './metadata/metadata.module';
+
+@Module({
+  imports: [DataSourceModule, MetadataModule],
+  exports: [DataSourceModule],
+})
+export class TenantModule {}