Feature: add createCustomField resolver (#1698)

* Feature: add createCustomField resolver

* update mocks

* fix import

* invalidate workspace datasource cache after migration

* fix typo

server/src/tenant/metadata/args/create-custom-field.input.ts

@@ -0,0 +1,21 @@
+import { ArgsType, Field } from '@nestjs/graphql';
+
+import { IsNotEmpty, IsString } from 'class-validator';
+
+@ArgsType()
+export class CreateCustomFieldInput {
+  @Field(() => String)
+  @IsNotEmpty()
+  @IsString()
+  name: string;
+
+  @Field(() => String)
+  @IsNotEmpty()
+  @IsString()
+  type: string;
+
+  @Field(() => String)
+  @IsNotEmpty()
+  @IsString()
+  objectId: string;
+}

server/src/tenant/metadata/data-source/data-source.service.ts

@@ -10,7 +10,7 @@ import { DataSource, QueryRunner, Table } from 'typeorm';
 import { EnvironmentService } from 'src/integrations/environment/environment.service';
 import { DataSourceMetadataService } from 'src/tenant/metadata/data-source-metadata/data-source-metadata.service';
 import { EntitySchemaGeneratorService } from 'src/tenant/metadata/entity-schema-generator/entity-schema-generator.service';
-import { TenantMigration } from 'src/tenant/metadata/migration-generator/tenant-migration.entity';
+import { TenantMigration } from 'src/tenant/metadata/tenant-migration/tenant-migration.entity';
 
 import { uuidToBase36 } from './data-source.util';
 
@@ -116,6 +116,8 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
       );
     }
 
+    // We only want the first one for now, we will handle multiple data sources later with remote datasources.
+    // However, we will need to differentiate the data sources because we won't run migrations on remote data sources for example.
     const dataSourceMetadata = dataSourcesMetadata[0];
     const schema = dataSourceMetadata.schema;
 

server/src/tenant/metadata/field-metadata/field-metadata.service.ts

@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 
 import { FieldMetadata } from './field-metadata.entity';
+import { generateColumnName } from './field-metadata.util';
 
 @Injectable()
 export class FieldMetadataService {
@@ -11,4 +12,29 @@ export class FieldMetadataService {
     @InjectRepository(FieldMetadata, 'metadata')
     private readonly fieldMetadataRepository: Repository<FieldMetadata>,
   ) {}
+
+  public async createFieldMetadata(
+    name: string,
+    type: string,
+    objectId: string,
+    workspaceId: string,
+  ): Promise<FieldMetadata> {
+    return await this.fieldMetadataRepository.save({
+      displayName: name,
+      type,
+      objectId,
+      isCustom: true,
+      targetColumnName: generateColumnName(name),
+      workspaceId,
+    });
+  }
+
+  public async getFieldMetadataByNameAndObjectId(
+    name: string,
+    objectId: string,
+  ): Promise<FieldMetadata | null> {
+    return await this.fieldMetadataRepository.findOne({
+      where: { displayName: name, objectId },
+    });
+  }
 }

server/src/tenant/metadata/field-metadata/field-metadata.util.ts

@@ -0,0 +1,9 @@
+/**
+ * Generate a column name from a field name removing unsupported characters.
+ *
+ * @param name string
+ * @returns string
+ */
+export function generateColumnName(name: string): string {
+  return name.toLowerCase().replace(/ /g, '_');
+}

server/src/tenant/metadata/metadata.module.ts

@@ -4,6 +4,7 @@ import { TypeOrmModule, TypeOrmModuleOptions } from '@nestjs/typeorm';
 import { MetadataService } from './metadata.service';
 import { MetadataController } from './metadata.controller';
 import { typeORMMetadataModuleOptions } from './metadata.datasource';
+import { MetadataResolver } from './metadata.resolver';
 
 import { DataSourceModule } from './data-source/data-source.module';
 import { DataSourceMetadataModule } from './data-source-metadata/data-source-metadata.module';
@@ -11,6 +12,7 @@ import { FieldMetadataModule } from './field-metadata/field-metadata.module';
 import { ObjectMetadataModule } from './object-metadata/object-metadata.module';
 import { EntitySchemaGeneratorModule } from './entity-schema-generator/entity-schema-generator.module';
 import { MigrationGeneratorModule } from './migration-generator/migration-generator.module';
+import { TenantMigrationModule } from './tenant-migration/tenant-migration.module';
 
 const typeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
   ...typeORMMetadataModuleOptions,
@@ -29,8 +31,9 @@ const typeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
     ObjectMetadataModule,
     EntitySchemaGeneratorModule,
     MigrationGeneratorModule,
+    TenantMigrationModule,
   ],
-  providers: [MetadataService],
+  providers: [MetadataService, MetadataResolver],
   exports: [MetadataService],
   controllers: [MetadataController],
 })

server/src/tenant/metadata/metadata.resolver.spec.ts

@@ -0,0 +1,26 @@
+import { Test, TestingModule } from '@nestjs/testing';
+
+import { MetadataResolver } from './metadata.resolver';
+import { MetadataService } from './metadata.service';
+
+describe('MetadataResolver', () => {
+  let resolver: MetadataResolver;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        MetadataResolver,
+        {
+          provide: MetadataService,
+          useValue: {},
+        },
+      ],
+    }).compile();
+
+    resolver = module.get<MetadataResolver>(MetadataResolver);
+  });
+
+  it('should be defined', () => {
+    expect(resolver).toBeDefined();
+  });
+});

server/src/tenant/metadata/metadata.resolver.ts

@@ -0,0 +1,30 @@
+import { UseGuards } from '@nestjs/common';
+import { Args, Mutation, Resolver } from '@nestjs/graphql';
+
+import { Workspace } from '@prisma/client';
+
+import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
+import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
+
+import { MetadataService } from './metadata.service';
+
+import { CreateCustomFieldInput } from './args/create-custom-field.input';
+
+@UseGuards(JwtAuthGuard)
+@Resolver()
+export class MetadataResolver {
+  constructor(private readonly metadataService: MetadataService) {}
+
+  @Mutation(() => String)
+  async createCustomField(
+    @Args() createCustomFieldInput: CreateCustomFieldInput,
+    @AuthWorkspace() workspace: Workspace,
+  ): Promise<string> {
+    return this.metadataService.createCustomField(
+      createCustomFieldInput.name,
+      createCustomFieldInput.objectId,
+      createCustomFieldInput.type,
+      workspace.id,
+    );
+  }
+}

server/src/tenant/metadata/metadata.service.spec.ts

@@ -2,12 +2,40 @@ import { Test, TestingModule } from '@nestjs/testing';
 
 import { MetadataService } from './metadata.service';
 
+import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
+import { DataSourceService } from './data-source/data-source.service';
+import { ObjectMetadataService } from './object-metadata/object-metadata.service';
+import { TenantMigrationService } from './tenant-migration/tenant-migration.service';
+import { FieldMetadataService } from './field-metadata/field-metadata.service';
+
 describe('MetadataService', () => {
   let service: MetadataService;
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [MetadataService],
+      providers: [
+        MetadataService,
+        {
+          provide: DataSourceService,
+          useValue: {},
+        },
+        {
+          provide: MigrationGeneratorService,
+          useValue: {},
+        },
+        {
+          provide: ObjectMetadataService,
+          useValue: {},
+        },
+        {
+          provide: TenantMigrationService,
+          useValue: {},
+        },
+        {
+          provide: FieldMetadataService,
+          useValue: {},
+        },
+      ],
     }).compile();
 
     service = module.get<MetadataService>(MetadataService);

server/src/tenant/metadata/metadata.service.ts

@@ -1,4 +1,99 @@
 import { Injectable } from '@nestjs/common';
 
+import { DataSourceService } from './data-source/data-source.service';
+import { FieldMetadataService } from './field-metadata/field-metadata.service';
+import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
+import { ObjectMetadataService } from './object-metadata/object-metadata.service';
+import { TenantMigrationService } from './tenant-migration/tenant-migration.service';
+import {
+  TenantMigrationColumnChange,
+  TenantMigrationTableChange,
+} from './tenant-migration/tenant-migration.entity';
+
 @Injectable()
-export class MetadataService {}
+export class MetadataService {
+  constructor(
+    private readonly dataSourceService: DataSourceService,
+    private readonly migrationGenerator: MigrationGeneratorService,
+    private readonly objectMetadataService: ObjectMetadataService,
+    private readonly tenantMigrationService: TenantMigrationService,
+    private readonly fieldMetadataService: FieldMetadataService,
+  ) {}
+
+  public async createCustomField(
+    name: string,
+    objectId: string,
+    type: string,
+    workspaceId: string,
+  ): Promise<string> {
+    const workspaceDataSource =
+      await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
+
+    if (!workspaceDataSource) {
+      throw new Error('Workspace data source not found');
+    }
+
+    const objectMetadata =
+      await this.objectMetadataService.getObjectMetadataFromId(objectId);
+
+    if (!objectMetadata) {
+      throw new Error('Object not found');
+    }
+
+    const fieldMetadataAlreadyExists =
+      await this.fieldMetadataService.getFieldMetadataByNameAndObjectId(
+        name,
+        objectId,
+      );
+
+    if (fieldMetadataAlreadyExists) {
+      throw new Error('Field already exists');
+    }
+
+    const createdFieldMetadata =
+      await this.fieldMetadataService.createFieldMetadata(
+        name,
+        type,
+        objectMetadata.id,
+        workspaceId,
+      );
+
+    await this.tenantMigrationService.createMigration(workspaceId, [
+      {
+        name: objectMetadata.targetTableName,
+        change: 'alter',
+        columns: [
+          {
+            name: createdFieldMetadata.targetColumnName,
+            type: this.convertMetadataTypeToColumnType(type),
+            change: 'create',
+          } satisfies TenantMigrationColumnChange,
+        ],
+      } satisfies TenantMigrationTableChange,
+    ]);
+
+    await this.migrationGenerator.executeMigrationFromPendingMigrations(
+      workspaceId,
+    );
+
+    return createdFieldMetadata.id;
+  }
+
+  private convertMetadataTypeToColumnType(type: string) {
+    switch (type) {
+      case 'text':
+      case 'url':
+      case 'phone':
+      case 'email':
+        return 'text';
+      case 'number':
+        return 'int';
+      case 'boolean':
+        return 'boolean';
+      case 'date':
+        return 'timestamp';
+      default:
+        throw new Error('Invalid type');
+    }
+  }
+}

server/src/tenant/metadata/migration-generator/migration-generator.module.ts

@@ -1,11 +1,12 @@
 import { Module } from '@nestjs/common';
 
 import { DataSourceModule } from 'src/tenant/metadata/data-source/data-source.module';
+import { TenantMigrationModule } from 'src/tenant/metadata/tenant-migration/tenant-migration.module';
 
 import { MigrationGeneratorService } from './migration-generator.service';
 
 @Module({
-  imports: [DataSourceModule],
+  imports: [DataSourceModule, TenantMigrationModule],
   exports: [MigrationGeneratorService],
   providers: [MigrationGeneratorService],
 })

server/src/tenant/metadata/migration-generator/migration-generator.service.spec.ts

@@ -1,6 +1,7 @@
 import { Test, TestingModule } from '@nestjs/testing';
 
 import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service';
+import { TenantMigrationService } from 'src/tenant/metadata/tenant-migration/tenant-migration.service';
 
 import { MigrationGeneratorService } from './migration-generator.service';
 
@@ -15,6 +16,10 @@ describe('MigrationGeneratorService', () => {
           provide: DataSourceService,
           useValue: {},
         },
+        {
+          provide: TenantMigrationService,
+          useValue: {},
+        },
       ],
     }).compile();
 

server/src/tenant/metadata/migration-generator/migration-generator.service.ts

@@ -1,40 +1,30 @@
 import { Injectable } from '@nestjs/common';
 
-import { IsNull, QueryRunner, Table, TableColumn } from 'typeorm';
+import { QueryRunner, Table, TableColumn } from 'typeorm';
 
 import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service';
-
 import {
-  Migration,
+  TenantMigrationTableChange,
-  MigrationColumn,
+  TenantMigrationColumnChange,
-  TenantMigration,
+} from 'src/tenant/metadata/tenant-migration/tenant-migration.entity';
-} from './tenant-migration.entity';
+import { TenantMigrationService } from 'src/tenant/metadata/tenant-migration/tenant-migration.service';
 
 @Injectable()
 export class MigrationGeneratorService {
-  constructor(private readonly dataSourceService: DataSourceService) {}
+  constructor(
+    private readonly dataSourceService: DataSourceService,
+    private readonly tenantMigrationService: TenantMigrationService,
+  ) {}
 
-  private async getPendingMigrations(workspaceId: string) {
+  /**
-    const workspaceDataSource =
+   * Executes pending migrations for a given workspace
-      await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
+   *
-
+   * @param workspaceId string
-    if (!workspaceDataSource) {
+   * @returns Promise<TenantMigrationTableChange[]>
-      throw new Error('Workspace data source not found');
+   */
-    }
+  public async executeMigrationFromPendingMigrations(
-
-    const tenantMigrationRepository =
-      workspaceDataSource.getRepository(TenantMigration);
-
-    return tenantMigrationRepository.find({
-      order: { createdAt: 'ASC' },
-      where: { appliedAt: IsNull() },
-    });
-  }
-
-  private async setAppliedAtForMigration(
     workspaceId: string,
-    migration: TenantMigration,
+  ): Promise<TenantMigrationTableChange[]> {
-  ) {
     const workspaceDataSource =
       await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
 
@@ -42,31 +32,13 @@ export class MigrationGeneratorService {
       throw new Error('Workspace data source not found');
     }
 
-    const tenantMigrationRepository =
+    const pendingMigrations =
-      workspaceDataSource.getRepository(TenantMigration);
+      await this.tenantMigrationService.getPendingMigrations(workspaceId);
 
-    await tenantMigrationRepository.save({
+    const flattenedPendingMigrations: TenantMigrationTableChange[] =
-      id: migration.id,
+      pendingMigrations.reduce((acc, pendingMigration) => {
-      appliedAt: new Date(),
-    });
-  }
-
-  public async executeMigrationFromPendingMigrations(workspaceId: string) {
-    const workspaceDataSource =
-      await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
-
-    if (!workspaceDataSource) {
-      throw new Error('Workspace data source not found');
-    }
-
-    const pendingMigrations = await this.getPendingMigrations(workspaceId);
-
-    const flattenedPendingMigrations: Migration[] = pendingMigrations.reduce(
-      (acc, pendingMigration) => {
         return [...acc, ...pendingMigration.migrations];
-      },
+      }, []);
-      [],
-    );
 
     const queryRunner = workspaceDataSource?.createQueryRunner();
     const schemaName = this.dataSourceService.getSchemaName(workspaceId);
@@ -80,16 +52,31 @@ export class MigrationGeneratorService {
     // Update appliedAt date for each migration
     // TODO: Should be done after the migration is successful
     pendingMigrations.forEach(async (pendingMigration) => {
-      await this.setAppliedAtForMigration(workspaceId, pendingMigration);
+      await this.tenantMigrationService.setAppliedAtForMigration(
+        workspaceId,
+        pendingMigration,
+      );
     });
 
+    await queryRunner.release();
+    // We want to destroy all connections to the workspace data source and invalidate the cache
+    // so that the next request will create a new connection and get the latest entities
+    await this.dataSourceService.disconnectFromWorkspaceDataSource(workspaceId);
+
     return flattenedPendingMigrations;
   }
 
+  /**
+   * Handles table changes for a given migration
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableMigration TenantMigrationTableChange
+   */
   private async handleTableChanges(
     queryRunner: QueryRunner,
     schemaName: string,
-    tableMigration: Migration,
+    tableMigration: TenantMigrationTableChange,
   ) {
     switch (tableMigration.change) {
       case 'create':
@@ -110,6 +97,13 @@ export class MigrationGeneratorService {
     }
   }
 
+  /**
+   * Creates a table for a given schema and table name
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableName string
+   */
   private async createTable(
     queryRunner: QueryRunner,
     schemaName: string,
@@ -137,11 +131,20 @@ export class MigrationGeneratorService {
     );
   }
 
+  /**
+   * Handles column changes for a given migration
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableName string
+   * @param columnMigrations TenantMigrationColumnChange[]
+   * @returns
+   */
   private async handleColumnChanges(
     queryRunner: QueryRunner,
     schemaName: string,
     tableName: string,
-    columnMigrations?: MigrationColumn[],
+    columnMigrations?: TenantMigrationColumnChange[],
   ) {
     if (!columnMigrations || columnMigrations.length === 0) {
       return;
@@ -169,11 +172,19 @@ export class MigrationGeneratorService {
     }
   }
 
+  /**
+   * Creates a column for a given schema, table name, and column migration
+   *
+   * @param queryRunner QueryRunner
+   * @param schemaName string
+   * @param tableName string
+   * @param migrationColumn TenantMigrationColumnChange
+   */
   private async createColumn(
     queryRunner: QueryRunner,
     schemaName: string,
     tableName: string,
-    migrationColumn: MigrationColumn,
+    migrationColumn: TenantMigrationColumnChange,
   ) {
     await queryRunner.addColumn(
       `${schemaName}.${tableName}`,

server/src/tenant/metadata/object-metadata/object-metadata.service.ts

@@ -18,4 +18,11 @@ export class ObjectMetadataService {
       relations: ['fields'],
     });
   }
+
+  public async getObjectMetadataFromId(objectMetadataId: string) {
+    return this.fieldMetadataRepository.findOne({
+      where: { id: objectMetadataId },
+      relations: ['fields'],
+    });
+  }
 }

server/src/tenant/metadata/tenant-migration/tenant-migration.entity.ts

@@ -5,16 +5,16 @@ import {
   PrimaryGeneratedColumn,
 } from 'typeorm';
 
-export type MigrationColumn = {
+export type TenantMigrationColumnChange = {
   name: string;
   type: string;
   change: 'create' | 'alter';
 };
 
-export type Migration = {
+export type TenantMigrationTableChange = {
   name: string;
   change: 'create' | 'alter';
-  columns?: MigrationColumn[];
+  columns?: TenantMigrationColumnChange[];
 };
 
 @Entity('tenant_migrations')
@@ -23,7 +23,7 @@ export class TenantMigration {
   id: string;
 
   @Column({ nullable: true, type: 'jsonb' })
-  migrations: Migration[];
+  migrations: TenantMigrationTableChange[];
 
   @Column({ nullable: true, name: 'applied_at' })
   appliedAt: Date;

server/src/tenant/metadata/tenant-migration/tenant-migration.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+
+import { DataSourceModule } from 'src/tenant/metadata/data-source/data-source.module';
+
+import { TenantMigrationService } from './tenant-migration.service';
+
+@Module({
+  imports: [DataSourceModule],
+  exports: [TenantMigrationService],
+  providers: [TenantMigrationService],
+})
+export class TenantMigrationModule {}

server/src/tenant/metadata/tenant-migration/tenant-migration.service.spec.ts

@@ -0,0 +1,25 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { TenantMigrationService } from './tenant-migration.service';
+import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service';
+
+describe('TenantMigrationService', () => {
+  let service: TenantMigrationService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        TenantMigrationService,
+        {
+          provide: DataSourceService,
+          useValue: {},
+        },
+      ],
+    }).compile();
+
+    service = module.get<TenantMigrationService>(TenantMigrationService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

server/src/tenant/metadata/tenant-migration/tenant-migration.service.ts

@@ -0,0 +1,92 @@
+import { Injectable } from '@nestjs/common';
+
+import { IsNull } from 'typeorm';
+
+import { DataSourceService } from 'src/tenant/metadata/data-source/data-source.service';
+
+import {
+  TenantMigration,
+  TenantMigrationTableChange,
+} from './tenant-migration.entity';
+
+@Injectable()
+export class TenantMigrationService {
+  constructor(private readonly dataSourceService: DataSourceService) {}
+
+  /**
+   * Get all pending migrations for a given workspaceId
+   *
+   * @param workspaceId: string
+   * @returns Promise<TenantMigration[]>
+   */
+  public async getPendingMigrations(
+    workspaceId: string,
+  ): Promise<TenantMigration[]> {
+    const workspaceDataSource =
+      await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
+
+    if (!workspaceDataSource) {
+      throw new Error('Workspace data source not found');
+    }
+
+    const tenantMigrationRepository =
+      workspaceDataSource.getRepository(TenantMigration);
+
+    return tenantMigrationRepository.find({
+      order: { createdAt: 'ASC' },
+      where: { appliedAt: IsNull() },
+    });
+  }
+
+  /**
+   * Set appliedAt as current date for a given migration.
+   * Should be called once the migration has been applied
+   *
+   * @param workspaceId: string
+   * @param migration: TenantMigration
+   */
+  public async setAppliedAtForMigration(
+    workspaceId: string,
+    migration: TenantMigration,
+  ) {
+    const workspaceDataSource =
+      await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
+
+    if (!workspaceDataSource) {
+      throw new Error('Workspace data source not found');
+    }
+
+    const tenantMigrationRepository =
+      workspaceDataSource.getRepository(TenantMigration);
+
+    await tenantMigrationRepository.save({
+      id: migration.id,
+      appliedAt: new Date(),
+    });
+  }
+
+  /**
+   * Create a new pending migration for a given workspaceId and expected changes
+   *
+   * @param workspaceId
+   * @param migrations
+   */
+  public async createMigration(
+    workspaceId: string,
+    migrations: TenantMigrationTableChange[],
+  ) {
+    const workspaceDataSource =
+      await this.dataSourceService.connectToWorkspaceDataSource(workspaceId);
+
+    if (!workspaceDataSource) {
+      throw new Error('Workspace data source not found');
+    }
+
+    const tenantMigrationRepository =
+      workspaceDataSource.getRepository(TenantMigration);
+
+    await tenantMigrationRepository.save({
+      migrations,
+    });
+  }
+}