Add targetColumnMap to FieldMetadata (#1863)

* Add targetColumnMap to FieldMetadata

* fix

* remove console.log

* fix test
This commit is contained in:
Weiko
2023-10-04 15:17:53 +02:00
committed by GitHub
parent 8f41792918
commit 42e8869e0e
22 changed files with 262 additions and 276 deletions

View File

@ -7,7 +7,7 @@ export class CreateCustomFieldInput {
@Field(() => String)
@IsNotEmpty()
@IsString()
name: string;
displayName: string;
@Field(() => String)
@IsNotEmpty()

View File

@ -1,12 +1,11 @@
import { Module } from '@nestjs/common';
import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
import { EntitySchemaGeneratorModule } from 'src/metadata/entity-schema-generator/entity-schema-generator.module';
import { DataSourceService } from './data-source.service';
@Module({
imports: [DataSourceMetadataModule, EntitySchemaGeneratorModule],
imports: [DataSourceMetadataModule],
providers: [DataSourceService],
exports: [DataSourceService],
})

View File

@ -2,7 +2,6 @@ import { Test, TestingModule } from '@nestjs/testing';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from 'src/metadata/entity-schema-generator/entity-schema-generator.service';
import { DataSourceService } from './data-source.service';
@ -23,10 +22,6 @@ describe('DataSourceService', () => {
provide: DataSourceMetadataService,
useValue: {},
},
{
provide: EntitySchemaGeneratorService,
useValue: {},
},
],
}).compile();

View File

@ -4,7 +4,6 @@ import { DataSource, QueryRunner, Table } from 'typeorm';
import { EnvironmentService } from 'src/integrations/environment/environment.service';
import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from 'src/metadata/entity-schema-generator/entity-schema-generator.service';
import { TenantMigration } from 'src/metadata/tenant-migration/tenant-migration.entity';
import { uuidToBase36 } from './data-source.util';
@ -17,7 +16,6 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
constructor(
private readonly environmentService: EnvironmentService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService,
) {
this.mainDataSource = new DataSource({
url: environmentService.getPGDatabaseUrl(),

View File

@ -1,15 +0,0 @@
export const baseColumns = {
id: {
primary: true,
type: 'uuid',
generated: 'uuid',
},
createdAt: {
type: 'timestamp',
createDate: true,
},
updatedAt: {
type: 'timestamp',
updateDate: true,
},
} as const;

View File

@ -1,12 +0,0 @@
import { Module } from '@nestjs/common';
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { EntitySchemaGeneratorService } from './entity-schema-generator.service';
@Module({
imports: [ObjectMetadataModule],
providers: [EntitySchemaGeneratorService],
exports: [EntitySchemaGeneratorService],
})
export class EntitySchemaGeneratorModule {}

View File

@ -1,29 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator.service';
describe('EntitySchemaGeneratorService', () => {
let service: EntitySchemaGeneratorService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
EntitySchemaGeneratorService,
{
provide: ObjectMetadataService,
useValue: {},
},
],
}).compile();
service = module.get<EntitySchemaGeneratorService>(
EntitySchemaGeneratorService,
);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -1,43 +0,0 @@
import { Injectable } from '@nestjs/common';
import { EntitySchema } from 'typeorm';
import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metadata.service';
import { baseColumns } from './base.entity';
import {
convertFieldTypeToPostgresType,
sanitizeColumnName,
} from './entity-schema-generator.util';
@Injectable()
export class EntitySchemaGeneratorService {
constructor(private readonly objectMetadataService: ObjectMetadataService) {}
async getTypeORMEntitiesByDataSourceId(dataSourceId: string) {
const objectMetadata =
await this.objectMetadataService.getObjectMetadataFromDataSourceId(
dataSourceId,
);
const entities = objectMetadata.map((object) => {
return new EntitySchema({
name: object.targetTableName,
columns: {
...baseColumns,
...object.fields.reduce((columns, field) => {
return {
...columns,
[sanitizeColumnName(field.targetColumnName)]: {
type: convertFieldTypeToPostgresType(field.type),
nullable: true,
},
};
}, {}),
},
});
});
return entities;
}
}

View File

@ -1,45 +0,0 @@
/**
* Converts a UUID to a base 36 string.
* This is used to generate the schema name since hyphens from workspace uuid are not allowed in postgres schema names.
*
* @param uuid
* @returns
*/
export const uuidToBase36 = (uuid: string): string => {
const hexString = uuid.replace(/-/g, '');
const base10Number = BigInt('0x' + hexString);
const base36String = base10Number.toString(36);
return base36String;
};
/**
* Sanitizes a column name by replacing all non-alphanumeric characters with an underscore.
* Note: Probablay not the best way to do this, leaving it here as a placeholder for now.
*
* @param columnName
* @returns string
*/
export const sanitizeColumnName = (columnName: string): string =>
columnName.replace(/[^a-zA-Z0-9]/g, '_');
/**
* Converts a field type to a postgres type. Field types are defined in the UI.
*
* @param fieldType
* @returns string
*/
export const convertFieldTypeToPostgresType = (fieldType: string): string => {
switch (fieldType) {
case 'text':
case 'url':
return 'text';
case 'number':
return 'numeric';
case 'boolean':
return 'boolean';
case 'date':
return 'timestamp';
default:
return 'text';
}
};

View File

@ -10,6 +10,10 @@ import {
import { ObjectMetadata } from 'src/metadata/object-metadata/object-metadata.entity';
export type FieldMetadataTargetColumnMap = {
[key: string]: string;
};
@Entity('field_metadata')
export class FieldMetadata {
@PrimaryGeneratedColumn('uuid')
@ -27,12 +31,27 @@ export class FieldMetadata {
@Column({ nullable: false, name: 'target_column_name' })
targetColumnName: string;
@Column({ nullable: true, name: 'description', type: 'text' })
description: string;
@Column({ nullable: true, name: 'icon' })
icon: string;
@Column({ nullable: true, name: 'placeholder' })
placeholder: string;
@Column({ nullable: true, name: 'target_column_map', type: 'jsonb' })
targetColumnMap: FieldMetadataTargetColumnMap;
@Column('text', { nullable: true, array: true })
enums: string[];
@Column({ default: false, name: 'is_custom' })
isCustom: boolean;
@Column({ default: false, name: 'is_active' })
isActive: boolean;
@Column({ nullable: true, default: true, name: 'is_nullable' })
isNullable: boolean;

View File

@ -4,7 +4,10 @@ import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { FieldMetadata } from './field-metadata.entity';
import { generateColumnName } from './field-metadata.util';
import {
generateColumnName,
generateTargetColumnMap,
} from './field-metadata.util';
@Injectable()
export class FieldMetadataService {
@ -14,22 +17,23 @@ export class FieldMetadataService {
) {}
public async createFieldMetadata(
name: string,
displayName: string,
type: string,
objectId: string,
workspaceId: string,
): Promise<FieldMetadata> {
return await this.fieldMetadataRepository.save({
displayName: name,
displayName: displayName,
type,
objectId,
isCustom: true,
targetColumnName: generateColumnName(name),
targetColumnName: generateColumnName(displayName), // deprecated
workspaceId,
targetColumnMap: generateTargetColumnMap(type),
});
}
public async getFieldMetadataByNameAndObjectId(
public async getFieldMetadataByDisplayNameAndObjectId(
name: string,
objectId: string,
): Promise<FieldMetadata | null> {

View File

@ -1,3 +1,9 @@
import { v4 } from 'uuid';
import { uuidToBase36 } from 'src/metadata/data-source/data-source.util';
import { FieldMetadataTargetColumnMap } from './field-metadata.entity';
/**
* Generate a column name from a field name removing unsupported characters.
*
@ -7,3 +13,38 @@
export function generateColumnName(name: string): string {
return name.toLowerCase().replace(/ /g, '_');
}
/**
* Generate a target column map for a given type, this is used to map the field to the correct column(s) in the database.
* This is used to support fields that map to multiple columns in the database.
*
* @param type string
* @returns FieldMetadataTargetColumnMap
*/
export function generateTargetColumnMap(
type: string,
): FieldMetadataTargetColumnMap {
switch (type) {
case 'text':
case 'phone':
case 'email':
case 'number':
case 'boolean':
case 'date':
return {
value: uuidToBase36(v4()),
};
case 'url':
return {
text: uuidToBase36(v4()),
link: uuidToBase36(v4()),
};
case 'money':
return {
amount: uuidToBase36(v4()),
currency: uuidToBase36(v4()),
};
default:
throw new Error(`Unknown type ${type}`);
}
}

View File

@ -1,42 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { MetadataController } from './metadata.controller';
import { DataSourceService } from './data-source/data-source.service';
import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service';
import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
describe('MetadataController', () => {
let controller: MetadataController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [MetadataController],
providers: [
{
provide: DataSourceService,
useValue: {},
},
{
provide: DataSourceMetadataService,
useValue: {},
},
{
provide: EntitySchemaGeneratorService,
useValue: {},
},
{
provide: MigrationGeneratorService,
useValue: {},
},
],
}).compile();
controller = module.get<MetadataController>(MetadataController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

View File

@ -1,54 +0,0 @@
import { Controller, Get, UseGuards } from '@nestjs/common';
import { Workspace } from '@prisma/client';
import { EntitySchema } from 'typeorm';
import { AuthWorkspace } from 'src/decorators/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { DataSourceMetadataService } from './data-source-metadata/data-source-metadata.service';
import { EntitySchemaGeneratorService } from './entity-schema-generator/entity-schema-generator.service';
import { DataSourceService } from './data-source/data-source.service';
import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
@UseGuards(JwtAuthGuard)
@Controller('metadata_legacy')
export class MetadataController {
constructor(
private readonly entitySchemaGeneratorService: EntitySchemaGeneratorService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
private readonly dataSourceService: DataSourceService,
private readonly migrationGenerator: MigrationGeneratorService,
) {}
@Get()
async getMetadata(@AuthWorkspace() workspace: Workspace) {
const dataSourcesMetadata =
await this.dataSourceMetadataService.getDataSourcesMetadataFromWorkspaceId(
workspace.id,
);
const entities: EntitySchema<{
id: unknown;
}>[] = [];
for (const dataSource of dataSourcesMetadata) {
const dataSourceEntities =
await this.entitySchemaGeneratorService.getTypeORMEntitiesByDataSourceId(
dataSource.id,
);
entities.push(...dataSourceEntities);
}
this.dataSourceService.createWorkspaceSchema(workspace.id);
await this.migrationGenerator.executeMigrationFromPendingMigrations(
workspace.id,
);
this.dataSourceService.connectToWorkspaceDataSource(workspace.id);
return entities;
}
}

View File

@ -5,8 +5,10 @@ import { GraphQLModule } from '@nestjs/graphql';
import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs';
import GraphQLJSON from 'graphql-type-json';
import { MigrationGeneratorModule } from 'src/metadata/migration-generator/migration-generator.module';
import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module';
import { MetadataService } from './metadata.service';
import { MetadataController } from './metadata.controller';
import { typeORMMetadataModuleOptions } from './metadata.datasource';
import { MetadataResolver } from './metadata.resolver';
@ -14,9 +16,6 @@ import { DataSourceModule } from './data-source/data-source.module';
import { DataSourceMetadataModule } from './data-source-metadata/data-source-metadata.module';
import { FieldMetadataModule } from './field-metadata/field-metadata.module';
import { ObjectMetadataModule } from './object-metadata/object-metadata.module';
import { EntitySchemaGeneratorModule } from './entity-schema-generator/entity-schema-generator.module';
import { MigrationGeneratorModule } from './migration-generator/migration-generator.module';
import { TenantMigrationModule } from './tenant-migration/tenant-migration.module';
const typeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
...typeORMMetadataModuleOptions,
@ -42,12 +41,10 @@ const typeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
DataSourceMetadataModule,
FieldMetadataModule,
ObjectMetadataModule,
EntitySchemaGeneratorModule,
MigrationGeneratorModule,
TenantMigrationModule,
],
providers: [MetadataService, MetadataResolver],
exports: [MetadataService],
controllers: [MetadataController],
})
export class MetadataModule {}

View File

@ -26,7 +26,7 @@ export class MetadataResolver {
@AuthWorkspace() workspace: Workspace,
): Promise<string> {
return this.metadataService.createCustomField(
createCustomFieldInput.name,
createCustomFieldInput.displayName,
createCustomFieldInput.objectId,
createCustomFieldInput.type,
workspace.id,

View File

@ -1,11 +1,12 @@
import { Test, TestingModule } from '@nestjs/testing';
import { MigrationGeneratorService } from 'src/metadata/migration-generator/migration-generator.service';
import { TenantMigrationService } from 'src/metadata/tenant-migration/tenant-migration.service';
import { MetadataService } from './metadata.service';
import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
import { DataSourceService } from './data-source/data-source.service';
import { ObjectMetadataService } from './object-metadata/object-metadata.service';
import { TenantMigrationService } from './tenant-migration/tenant-migration.service';
import { FieldMetadataService } from './field-metadata/field-metadata.service';
describe('MetadataService', () => {

View File

@ -1,14 +1,17 @@
import { Injectable } from '@nestjs/common';
import { DataSourceService } from './data-source/data-source.service';
import { FieldMetadataService } from './field-metadata/field-metadata.service';
import { MigrationGeneratorService } from './migration-generator/migration-generator.service';
import { ObjectMetadataService } from './object-metadata/object-metadata.service';
import { TenantMigrationService } from './tenant-migration/tenant-migration.service';
import { MigrationGeneratorService } from 'src/metadata/migration-generator/migration-generator.service';
import { TenantMigrationService } from 'src/metadata/tenant-migration/tenant-migration.service';
import {
TenantMigrationColumnChange,
TenantMigrationTableChange,
} from './tenant-migration/tenant-migration.entity';
} from 'src/metadata/tenant-migration/tenant-migration.entity';
import { convertFieldMetadataToColumnChanges } from './metadata.util';
import { DataSourceService } from './data-source/data-source.service';
import { FieldMetadataService } from './field-metadata/field-metadata.service';
import { ObjectMetadataService } from './object-metadata/object-metadata.service';
@Injectable()
export class MetadataService {
@ -21,7 +24,7 @@ export class MetadataService {
) {}
public async createCustomField(
name: string,
displayName: string,
objectId: string,
type: string,
workspaceId: string,
@ -41,8 +44,8 @@ export class MetadataService {
}
const fieldMetadataAlreadyExists =
await this.fieldMetadataService.getFieldMetadataByNameAndObjectId(
name,
await this.fieldMetadataService.getFieldMetadataByDisplayNameAndObjectId(
displayName,
objectId,
);
@ -52,7 +55,7 @@ export class MetadataService {
const createdFieldMetadata =
await this.fieldMetadataService.createFieldMetadata(
name,
displayName,
type,
objectMetadata.id,
workspaceId,
@ -63,6 +66,8 @@ export class MetadataService {
name: objectMetadata.targetTableName,
change: 'alter',
columns: [
...convertFieldMetadataToColumnChanges(createdFieldMetadata),
// Deprecated
{
name: createdFieldMetadata.targetColumnName,
type: this.convertMetadataTypeToColumnType(type),
@ -79,6 +84,7 @@ export class MetadataService {
return createdFieldMetadata.id;
}
// Deprecated with target_column_name
private convertMetadataTypeToColumnType(type: string) {
switch (type) {
case 'text':
@ -92,6 +98,8 @@ export class MetadataService {
return 'boolean';
case 'date':
return 'timestamp';
case 'money':
return 'integer';
default:
throw new Error('Invalid type');
}

View File

@ -0,0 +1,79 @@
import { TenantMigrationColumnChange } from 'src/metadata/tenant-migration/tenant-migration.entity';
import { FieldMetadata } from './field-metadata/field-metadata.entity';
export function convertFieldMetadataToColumnChanges(
fieldMetadata: FieldMetadata,
): TenantMigrationColumnChange[] {
switch (fieldMetadata.type) {
case 'text':
return [
{
name: fieldMetadata.targetColumnMap.value,
change: 'create',
type: 'text',
},
];
case 'phone':
case 'email':
return [
{
name: fieldMetadata.targetColumnMap.value,
change: 'create',
type: 'varchar',
},
];
case 'number':
return [
{
name: fieldMetadata.targetColumnMap.value,
change: 'create',
type: 'integer',
},
];
case 'boolean':
return [
{
name: fieldMetadata.targetColumnMap.value,
change: 'create',
type: 'boolean',
},
];
case 'date':
return [
{
name: fieldMetadata.targetColumnMap.value,
change: 'create',
type: 'timestamp',
},
];
case 'url':
return [
{
name: fieldMetadata.targetColumnMap.text,
change: 'create',
type: 'varchar',
},
{
name: fieldMetadata.targetColumnMap.link,
change: 'create',
type: 'varchar',
},
];
case 'money':
return [
{
name: fieldMetadata.targetColumnMap.amount,
change: 'create',
type: 'integer',
},
{
name: fieldMetadata.targetColumnMap.currency,
change: 'create',
type: 'varchar',
},
];
default:
throw new Error(`Unknown type ${fieldMetadata.type}`);
}
}

View File

@ -0,0 +1,71 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddTargetColumnMap1696409050890 implements MigrationInterface {
name = 'AddTargetColumnMap1696409050890';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD "description" text`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD "icon" character varying`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD "placeholder" character varying`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD "target_column_map" jsonb`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" ADD "is_active" boolean NOT NULL DEFAULT false`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" ADD "display_name_singular" character varying`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" ADD "display_name_plural" character varying`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" ADD "description" text`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" ADD "icon" character varying`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" ADD "is_active" boolean NOT NULL DEFAULT false`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" DROP COLUMN "is_active"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" DROP COLUMN "icon"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" DROP COLUMN "description"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" DROP COLUMN "display_name_plural"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."object_metadata" DROP COLUMN "display_name_singular"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP COLUMN "is_active"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP COLUMN "target_column_map"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP COLUMN "placeholder"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP COLUMN "icon"`,
);
await queryRunner.query(
`ALTER TABLE "metadata"."field_metadata" DROP COLUMN "description"`,
);
}
}

View File

@ -17,15 +17,31 @@ export class ObjectMetadata {
@Column({ nullable: false, name: 'data_source_id' })
dataSourceId: string;
// Deprecated
@Column({ nullable: false, name: 'display_name' })
displayName: string;
@Column({ nullable: true, name: 'display_name_singular' })
displayNameSingular: string;
@Column({ nullable: true, name: 'display_name_plural' })
displayNamePlural: string;
@Column({ nullable: true, name: 'description', type: 'text' })
description: string;
@Column({ nullable: true, name: 'icon' })
icon: string;
@Column({ nullable: false, name: 'target_table_name' })
targetTableName: string;
@Column({ default: false, name: 'is_custom' })
isCustom: boolean;
@Column({ default: false, name: 'is_active' })
isActive: boolean;
@Column({ nullable: false, name: 'workspace_id' })
workspaceId: string;

View File

@ -3,7 +3,6 @@ import { Module } from '@nestjs/common';
import { EntityResolverModule } from 'src/tenant/entity-resolver/entity-resolver.module';
import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
import { EntitySchemaGeneratorModule } from 'src/metadata/entity-schema-generator/entity-schema-generator.module';
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { SchemaGenerationService } from './schema-generation.service';
@ -12,7 +11,6 @@ import { SchemaGenerationService } from './schema-generation.service';
imports: [
EntityResolverModule,
DataSourceMetadataModule,
EntitySchemaGeneratorModule,
ObjectMetadataModule,
],
providers: [SchemaGenerationService, JwtAuthGuard],