admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: dynamic graphQL schema generation based on user workspace (#1725)

Browse Source 
* wip: refacto and start creating custom resolver

* feat: findMany & findUnique of a custom entity

* feat: wip pagination

* feat: initial metadata migration

* feat: universal findAll with pagination

* fix: clean small stuff in pagination

* fix: test

* fix: miss file

* feat: rename custom into universal

* feat: create metadata schema in default database

* Multi-tenant db schemas POC

fix tests and use query builders

remove synchronize

restore updatedAt

remove unnecessary import

use queryRunner

fix camelcase

add migrations for standard objects

Multi-tenant db schemas POC

fix tests and use query builders

remove synchronize

restore updatedAt

remove unnecessary import

use queryRunner

fix camelcase

add migrations for standard objects

poc: conditional schema at runtime

wip: try to create resolver in Nest.JS context

fix

* feat: wip add pg_graphql

* feat: setup pg_graphql during database init

* wip: dynamic resolver

* poc: dynamic resolver and query using pg_graphql

* feat: pg_graphql use ARG in Dockerfile

* feat: clean findMany & findOne dynamic resolver

* feat: get correct schema based on access token

* fix: remove old file

* fix: tests

* fix: better comment

* fix: e2e test not working, error format change due to yoga

* remove typeorm entity generation + fix jwt + fix search_path + remove anon

* fix conflict

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
Co-authored-by: corentin <corentin@twenty.com>
This commit is contained in:
Jérémy M
2023-09-28 16:27:34 +02:00
committed by GitHub
parent 485bc64b4f
commit 629bdbbf50
35 changed files with 1860 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
server/src/tenant/metadata/data-source-metadata/data-source-metadata.service.ts
View File

@ -28,10 +28,17 @@ export class DataSourceMetadataService {
});
}
getDataSourcesMetadataFromWorkspaceId(workspaceId: string) {
async getDataSourcesMetadataFromWorkspaceId(workspaceId: string) {
return this.dataSourceMetadataRepository.find({
where: { workspaceId },
order: { createdAt: 'DESC' },
});
}
async getLastDataSourceMetadataFromWorkspaceIdOrFail(workspaceId: string) {
return this.dataSourceMetadataRepository.findOneOrFail({
where: { workspaceId },
order: { createdAt: 'DESC' },
});
}
}

43
server/src/tenant/metadata/data-source/data-source.service.ts
View File

@ -1,9 +1,4 @@
import {
Injectable,
NotFoundException,
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { DataSource, QueryRunner, Table } from 'typeorm';
@ -37,15 +32,14 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
* @param workspaceId
* @returns Promise<void>
*/
public async createWorkspaceSchema(workspaceId: string): Promise<void> {
public async createWorkspaceSchema(workspaceId: string): Promise<string> {
const schemaName = this.getSchemaName(workspaceId);
const queryRunner = this.mainDataSource.createQueryRunner();
const schemaAlreadyExists = await queryRunner.hasSchema(schemaName);
if (schemaAlreadyExists) {
throw new Error(
`Schema ${schemaName} already exists for workspace ${workspaceId}`,
);
return schemaName;
}
await queryRunner.createSchema(schemaName, true);
@ -56,6 +50,8 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
workspaceId,
schemaName,
);
return schemaName;
}
private async createMigrationTable(
@ -105,20 +101,12 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
return cachedDataSource;
}
const dataSourcesMetadata =
await this.dataSourceMetadataService.getDataSourcesMetadataFromWorkspaceId(
workspaceId,
);
if (dataSourcesMetadata.length === 0) {
throw new NotFoundException(
`We can't find any data source for this workspace id (${workspaceId}).`,
);
}
// We only want the first one for now, we will handle multiple data sources later with remote datasources.
// However, we will need to differentiate the data sources because we won't run migrations on remote data sources for example.
const dataSourceMetadata = dataSourcesMetadata[0];
const dataSourceMetadata =
await this.dataSourceMetadataService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
workspaceId,
);
const schema = dataSourceMetadata.schema;
// Probably not needed as we will ask for the schema name OR store public by default if it's remote
@ -128,11 +116,6 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
);
}
const entities =
await this.entitySchemaGeneratorService.getTypeORMEntitiesByDataSourceId(
dataSourceMetadata.id,
);
const workspaceDataSource = new DataSource({
// TODO: We should use later dataSourceMetadata.type and use a switch case condition to create the right data source
url: dataSourceMetadata.url ?? this.environmentService.getPGDatabaseUrl(),
@ -141,15 +124,17 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
schema,
entities: {
TenantMigration,
...entities,
},
});
await workspaceDataSource.initialize();
// Set search path to workspace schema for raw queries
await workspaceDataSource?.query(`SET search_path TO ${schema};`);
this.dataSources.set(workspaceId, workspaceDataSource);
return this.dataSources.get(workspaceId);
return workspaceDataSource;
}
/**

1
server/src/tenant/metadata/entity-schema-generator/entity-schema-generator.util.ts
View File

@ -31,7 +31,6 @@ export const sanitizeColumnName = (columnName: string): string =>
export const convertFieldTypeToPostgresType = (fieldType: string): string => {
switch (fieldType) {
case 'text':
return 'text';
case 'url':
return 'text';
case 'number':

6
server/src/tenant/metadata/field-metadata/field-metadata.entity.ts
View File

@ -27,9 +27,15 @@ export class FieldMetadata {
@Column({ nullable: false, name: 'target_column_name' })
targetColumnName: string;
@Column('text', { nullable: true, array: true })
enums: string[];
@Column({ default: false, name: 'is_custom' })
isCustom: boolean;
@Column({ nullable: true, default: true, name: 'is_nullable' })
isNullable: boolean;
@Column({ nullable: false, name: 'workspace_id' })
workspaceId: string;

2
server/src/tenant/metadata/metadata.controller.ts
View File

@ -41,6 +41,8 @@ export class MetadataController {
entities.push(...dataSourceEntities);
}
this.dataSourceService.createWorkspaceSchema(workspace.id);
await this.migrationGenerator.executeMigrationFromPendingMigrations(
workspace.id,
);

5
server/src/tenant/metadata/migration-generator/migration-generator.service.ts
View File

@ -58,11 +58,6 @@ export class MigrationGeneratorService {
);
});
await queryRunner.release();
// We want to destroy all connections to the workspace data source and invalidate the cache
// so that the next request will create a new connection and get the latest entities
await this.dataSourceService.disconnectFromWorkspaceDataSource(workspaceId);
return flattenedPendingMigrations;
}

61
server/src/tenant/metadata/migrations/1695717691800-alter-field-metadata-table.ts Normal file
View File

@ -0,0 +1,61 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AlterFieldMetadataTable1695717691800
implements MigrationInterface
{
name = 'AlterFieldMetadataTable1695717691800';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD "enums" text array`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD "is_nullable" boolean DEFAULT true`,
);
await queryRunner.query(
`ALTER TYPE "metadata"."data_source_metadata_type_enum" RENAME TO "data_source_metadata_type_enum_old"`,
);
await queryRunner.query(
`CREATE TYPE "metadata"."data_source_metadata_type_enum" AS ENUM('postgres')`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."data_source_metadata" ALTER COLUMN "type" DROP DEFAULT`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."data_source_metadata" ALTER COLUMN "type" TYPE "metadata"."data_source_metadata_type_enum" USING "type"::"text"::"metadata"."data_source_metadata_type_enum"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."data_source_metadata" ALTER COLUMN "type" SET DEFAULT 'postgres'`,
);
await queryRunner.query(
`DROP TYPE "metadata"."data_source_metadata_type_enum_old"`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TYPE "metadata"."data_source_metadata_type_enum_old" AS ENUM('postgres', 'mysql')`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."data_source_metadata" ALTER COLUMN "type" DROP DEFAULT`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."data_source_metadata" ALTER COLUMN "type" TYPE "metadata"."data_source_metadata_type_enum_old" USING "type"::"text"::"metadata"."data_source_metadata_type_enum_old"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."data_source_metadata" ALTER COLUMN "type" SET DEFAULT 'postgres'`,
);
await queryRunner.query(
`DROP TYPE "metadata"."data_source_metadata_type_enum"`,
);
await queryRunner.query(
`ALTER TYPE "metadata"."data_source_metadata_type_enum_old" RENAME TO "data_source_metadata_type_enum"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP COLUMN "is_nullable"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP COLUMN "enums"`,
);
}
}

4
server/src/tenant/metadata/tenant-migration/tenant-migration.service.spec.ts
View File

@ -1,7 +1,9 @@
import { Test, TestingModule } from '@nestjs/testing';
import { TenantMigrationService } from './tenant-migration.service';
import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service';
import { TenantMigrationService } from './tenant-migration.service';
describe('TenantMigrationService', () => {
let service: TenantMigrationService;