Convert metadata tables to camelCase (#2400)

* Convert metadata tables to camelCase

* datasourcemetadataid to datasourceid

* refactor metadata folders

* fix command

* move commands out of metadata

* fix seed

* rename objectId and fieldId in objectMetadataId and fieldMetadataId in FE

* fix field-metadata

* Fix

* Fix

* remove logs

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Weiko
2023-11-09 20:06:10 +01:00
committed by GitHub
parent 5622f42e7a
commit 1cf08c797f
238 changed files with 1851 additions and 2252 deletions

View File

@ -0,0 +1,48 @@
import { Command, CommandRunner, Option } from 'nest-commander';
import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
import { TenantManagerService } from 'src/tenant-manager/tenant-manager.service';
// TODO: implement dry-run
interface RunTenantMigrationsOptions {
workspaceId: string;
}
@Command({
name: 'tenant:sync-metadata',
description: 'Sync metadata',
})
export class SyncTenantMetadataCommand extends CommandRunner {
constructor(
private readonly tenantManagerService: TenantManagerService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
) {
super();
}
async run(
_passedParam: string[],
options: RunTenantMigrationsOptions,
): Promise<void> {
// TODO: run in a dedicated job + run queries in a transaction.
const dataSourceMetadata =
await this.dataSourceMetadataService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
options.workspaceId,
);
// TODO: This solution could be improved, using a diff for example, we should not have to delete all metadata and recreate them.
await this.tenantManagerService.resetStandardObjectsAndFieldsMetadata(
dataSourceMetadata.id,
options.workspaceId,
);
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id',
required: true,
})
parseWorkspaceId(value: string): string {
return value;
}
}

View File

@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { TenantManagerModule } from 'src/tenant-manager/tenant-manager.module';
import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
import { SyncTenantMetadataCommand } from './sync-tenant-metadata.command';
@Module({
imports: [TenantManagerModule, DataSourceMetadataModule],
providers: [SyncTenantMetadataCommand],
})
export class TenantManagerCommandsModule {}

View File

@ -0,0 +1,57 @@
const companiesMetadata = {
nameSingular: 'companyV2',
namePlural: 'companiesV2',
labelSingular: 'Company',
labelPlural: 'Companies',
targetTableName: 'company',
description: 'A company',
icon: 'IconBuildingSkyscraper',
fields: [
{
type: 'TEXT',
name: 'name',
label: 'Name',
targetColumnMap: {
value: 'name',
},
description: 'Name of the company',
icon: 'IconBuildingSkyscraper',
isNullable: false,
},
{
type: 'TEXT',
name: 'domainName',
label: 'Domain Name',
targetColumnMap: {
value: 'domainName',
},
description: 'Domain name of the company',
icon: 'IconLink',
isNullable: true,
},
{
type: 'TEXT',
name: 'address',
label: 'Address',
targetColumnMap: {
value: 'address',
},
description: 'Address of the company',
icon: 'IconMap',
isNullable: true,
},
{
type: 'NUMBER',
name: 'employees',
label: 'Employees',
targetColumnMap: {
value: 'employees',
},
description: 'Number of employees',
icon: 'IconUsers',
isNullable: true,
},
],
};
export default companiesMetadata;

View File

@ -0,0 +1,13 @@
import companiesMetadata from './companies/companies.metadata';
import viewFieldsMetadata from './view-fields/view-fields.metadata';
import viewFiltersMetadata from './view-filters/view-filters.metadata';
import viewSortsMetadata from './view-sorts/view-sorts.metadata';
import viewsMetadata from './views/views.metadata';
export const standardObjectsMetadata = {
companyV2: companiesMetadata,
viewV2: viewsMetadata,
viewFieldV2: viewFieldsMetadata,
viewFilterV2: viewFiltersMetadata,
viewSortV2: viewSortsMetadata,
};

View File

@ -0,0 +1,68 @@
const viewFieldsMetadata = {
nameSingular: 'viewFieldV2',
namePlural: 'viewFieldsV2',
labelSingular: 'View Field',
labelPlural: 'View Fields',
targetTableName: 'viewField',
description: '(System) View Fields',
icon: 'IconColumns3',
fields: [
{
type: 'TEXT',
name: 'fieldMetadataId',
label: 'Field Id',
targetColumnMap: {
value: 'fieldMetadataId',
},
description: 'View Field target field',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'viewId',
label: 'View Id',
targetColumnMap: {
value: 'viewId',
},
description: 'View Field related view',
icon: null,
isNullable: false,
},
{
type: 'BOOLEAN',
name: 'isVisible',
label: 'Visible',
targetColumnMap: {
value: 'isVisible',
},
description: 'View Field visibility',
icon: null,
isNullable: false,
},
{
type: 'NUMBER',
name: 'size',
label: 'Size',
targetColumnMap: {
value: 'size',
},
description: 'View Field size',
icon: null,
isNullable: false,
},
{
type: 'NUMBER',
name: 'position',
label: 'Position',
targetColumnMap: {
value: 'position',
},
description: 'View Field position',
icon: null,
isNullable: false,
},
],
};
export default viewFieldsMetadata;

View File

@ -0,0 +1,68 @@
const viewFiltersMetadata = {
nameSingular: 'viewFilterV2',
namePlural: 'viewFiltersV2',
labelSingular: 'View Filter',
labelPlural: 'View Filters',
targetTableName: 'viewFilter',
description: '(System) View Filters',
icon: 'IconFilterBolt',
fields: [
{
type: 'TEXT',
name: 'fieldMetadataId',
label: 'Field Id',
targetColumnMap: {
value: 'fieldMetadataId',
},
description: 'View Filter target field',
icon: null,
isNullable: true,
},
{
type: 'TEXT',
name: 'viewId',
label: 'View Id',
targetColumnMap: {
value: 'viewId',
},
description: 'View Filter related view',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'operand',
label: 'Operand',
targetColumnMap: {
value: 'operand',
},
description: 'View Filter operand',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'value',
label: 'Value',
targetColumnMap: {
value: 'value',
},
description: 'View Filter value',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'displayValue',
label: 'Display Value',
targetColumnMap: {
value: 'displayValue',
},
description: 'View Filter Display Value',
icon: null,
isNullable: false,
},
],
};
export default viewFiltersMetadata;

View File

@ -0,0 +1,46 @@
const viewSortsMetadata = {
nameSingular: 'viewSortV2',
namePlural: 'viewSortsV2',
labelSingular: 'View Sort',
labelPlural: 'View Sorts',
targetTableName: 'viewSort',
description: '(System) View Sorts',
icon: 'IconArrowsSort',
fields: [
{
type: 'TEXT',
name: 'fieldMetadataId',
label: 'Field Id',
targetColumnMap: {
value: 'fieldMetadataId',
},
description: 'View Sort target field',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'viewId',
label: 'View Id',
targetColumnMap: {
value: 'viewId',
},
description: 'View Sort related view',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'direction',
label: 'Direction',
targetColumnMap: {
value: 'direction',
},
description: 'View Sort direction',
icon: null,
isNullable: false,
},
],
};
export default viewSortsMetadata;

View File

@ -0,0 +1,46 @@
const viewsMetadata = {
nameSingular: 'viewV2',
namePlural: 'viewsV2',
labelSingular: 'View',
labelPlural: 'Views',
targetTableName: 'view',
description: '(System) Views',
icon: 'IconLayoutCollage',
fields: [
{
type: 'TEXT',
name: 'name',
label: 'Name',
targetColumnMap: {
value: 'name',
},
description: 'View name',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'objectMetadataId',
label: 'Object Id',
targetColumnMap: {
value: 'objectMetadataId',
},
description: 'View target object',
icon: null,
isNullable: false,
},
{
type: 'TEXT',
name: 'type',
label: 'Type',
targetColumnMap: {
value: 'type',
},
description: 'View type',
icon: null,
isNullable: false,
},
],
};
export default viewsMetadata;

View File

@ -0,0 +1,269 @@
import { DataSource, EntityManager } from 'typeorm';
export const standardObjectsPrefillData = async (
workspaceDataSource: DataSource,
schemaName: string,
) => {
workspaceDataSource.transaction(async (entityManager: EntityManager) => {
const createdCompanies = await entityManager
.createQueryBuilder()
.insert()
.into(`${schemaName}.company`, [
'name',
'domainName',
'address',
'employees',
])
.orIgnore()
.values([
{
name: 'Airbnb',
domainName: 'airbnb.com',
address: 'San Francisco',
employees: 5000,
},
{
name: 'Qonto',
domainName: 'qonto.com',
address: 'San Francisco',
employees: 800,
},
{
name: 'Stripe',
domainName: 'stripe.com',
address: 'San Francisco',
employees: 8000,
},
{
name: 'Figma',
domainName: 'figma.com',
address: 'San Francisco',
employees: 800,
},
{
name: 'Notion',
domainName: 'notion.com',
address: 'San Francisco',
employees: 400,
},
])
.returning('*')
.execute();
const companyIdMap = createdCompanies.raw.reduce((acc, view) => {
acc[view.name] = view.id;
return acc;
}, {});
const createdViews = await entityManager
.createQueryBuilder()
.insert()
.into(`${schemaName}.view`, ['name', 'objectMetadataId', 'type'])
.orIgnore()
.values([
{
name: 'All companies',
objectMetadataId: 'company',
type: 'table',
},
{
name: 'All people',
objectMetadataId: 'person',
type: 'table',
},
{
name: 'All opportunities',
objectMetadataId: 'company',
type: 'kanban',
},
{
name: 'All Companies (V2)',
objectMetadataId: companyIdMap['Airbnb'],
type: 'table',
},
])
.returning('*')
.execute();
const viewIdMap = createdViews.raw.reduce((acc, view) => {
acc[view.name] = view.id;
return acc;
}, {});
await entityManager
.createQueryBuilder()
.insert()
.into(`${schemaName}.viewField`, [
'fieldMetadataId',
'viewId',
'position',
'isVisible',
'size',
])
.orIgnore()
.values([
{
fieldMetadataId: 'name',
viewId: viewIdMap['All Companies (V2)'],
position: 0,
isVisible: true,
size: 180,
},
{
fieldMetadataId: 'name',
viewId: viewIdMap['All companies'],
position: 0,
isVisible: true,
size: 180,
},
{
fieldMetadataId: 'domainName',
viewId: viewIdMap['All companies'],
position: 1,
isVisible: true,
size: 100,
},
{
fieldMetadataId: 'accountOwner',
viewId: viewIdMap['All companies'],
position: 2,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'createdAt',
viewId: viewIdMap['All companies'],
position: 3,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'employees',
viewId: viewIdMap['All companies'],
position: 4,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'linkedin',
viewId: viewIdMap['All companies'],
position: 5,
isVisible: true,
size: 170,
},
{
fieldMetadataId: 'address',
viewId: viewIdMap['All companies'],
position: 6,
isVisible: true,
size: 170,
},
{
fieldMetadataId: 'displayName',
viewId: viewIdMap['All people'],
position: 0,
isVisible: true,
size: 210,
},
{
fieldMetadataId: 'email',
viewId: viewIdMap['All people'],
position: 1,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'company',
viewId: viewIdMap['All people'],
position: 2,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'phone',
viewId: viewIdMap['All people'],
position: 3,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'createdAt',
viewId: viewIdMap['All people'],
position: 4,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'city',
viewId: viewIdMap['All people'],
position: 5,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'jobTitle',
viewId: viewIdMap['All people'],
position: 6,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'linkedin',
viewId: viewIdMap['All people'],
position: 7,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'x',
viewId: viewIdMap['All people'],
position: 8,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'amount',
viewId: viewIdMap['All opportunities'],
position: 0,
isVisible: true,
size: 180,
},
{
fieldMetadataId: 'probability',
viewId: viewIdMap['All opportunities'],
position: 1,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'closeDate',
viewId: viewIdMap['All opportunities'],
position: 2,
isVisible: true,
size: 100,
},
{
fieldMetadataId: 'company',
viewId: viewIdMap['All opportunities'],
position: 3,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'createdAt',
viewId: viewIdMap['All opportunities'],
position: 4,
isVisible: true,
size: 150,
},
{
fieldMetadataId: 'pointOfContact',
viewId: viewIdMap['All opportunities'],
position: 5,
isVisible: true,
size: 150,
},
])
.execute();
});
};

View File

@ -0,0 +1,24 @@
import { Module } from '@nestjs/common';
import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module';
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module';
import { TenantDataSourceModule } from 'src/tenant-datasource/tenant-datasource.module';
import { TenantMigrationRunnerModule } from 'src/tenant-migration-runner/tenant-migration-runner.module';
import { TenantManagerService } from './tenant-manager.service';
@Module({
imports: [
TenantDataSourceModule,
TenantMigrationModule,
TenantMigrationRunnerModule,
ObjectMetadataModule,
FieldMetadataModule,
DataSourceMetadataModule,
],
exports: [TenantManagerService],
providers: [TenantManagerService],
})
export class TenantManagerModule {}

View File

@ -0,0 +1,143 @@
import { Injectable } from '@nestjs/common';
import { DataSourceEntity } from 'src/database/typeorm/metadata/entities/data-source.entity';
import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
import { FieldMetadataService } from 'src/metadata/field-metadata/field-metadata.service';
import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metadata.service';
import { TenantMigrationRunnerService } from 'src/tenant-migration-runner/tenant-migration-runner.service';
import { TenantMigrationService } from 'src/metadata/tenant-migration/tenant-migration.service';
import { standardObjectsPrefillData } from 'src/tenant-manager/standard-objects-prefill-data/standard-objects-prefill-data';
import { TenantDataSourceService } from 'src/tenant-datasource/tenant-datasource.service';
import { standardObjectsMetadata } from 'src/tenant-manager/standard-objects-metadata/standard-object-metadata';
@Injectable()
export class TenantManagerService {
constructor(
private readonly tenantDataSourceService: TenantDataSourceService,
private readonly tenantMigrationService: TenantMigrationService,
private readonly migrationRunnerService: TenantMigrationRunnerService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly fieldMetadataService: FieldMetadataService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
) {}
/**
* Init a workspace by creating a new data source and running all migrations
* @param workspaceId
* @returns Promise<void>
*/
public async init(workspaceId: string): Promise<void> {
const schemaName =
await this.tenantDataSourceService.createWorkspaceDBSchema(workspaceId);
const dataSourceMetadata =
await this.dataSourceMetadataService.createDataSourceMetadata(
workspaceId,
schemaName,
);
await this.tenantMigrationService.insertStandardMigrations(workspaceId);
await this.migrationRunnerService.executeMigrationFromPendingMigrations(
workspaceId,
);
await this.createStandardObjectsAndFieldsMetadata(
dataSourceMetadata.id,
workspaceId,
);
await this.prefillWorkspaceWithStandardObjects(
dataSourceMetadata,
workspaceId,
);
}
/**
*
* Create all standard objects and fields metadata for a given workspace
*
* @param dataSourceId
* @param workspaceId
*/
public async createStandardObjectsAndFieldsMetadata(
dataSourceId: string,
workspaceId: string,
) {
await this.objectMetadataService.createMany(
Object.values(standardObjectsMetadata).map((objectMetadata) => ({
...objectMetadata,
dataSourceId,
workspaceId,
isCustom: false,
isActive: true,
fields: objectMetadata.fields.map((field) => ({
...field,
workspaceId,
isCustom: false,
isActive: true,
})),
})),
);
}
/**
*
* Reset all standard objects and fields metadata for a given workspace
*
* @param dataSourceId
* @param workspaceId
*/
public async resetStandardObjectsAndFieldsMetadata(
dataSourceId: string,
workspaceId: string,
) {
await this.objectMetadataService.deleteMany({
workspaceId: { eq: workspaceId },
});
await this.createStandardObjectsAndFieldsMetadata(
dataSourceId,
workspaceId,
);
}
/**
*
* We are prefilling a few standard objects with data to make it easier for the user to get started.
*
* @param dataSourceMetadata
* @param workspaceId
*/
private async prefillWorkspaceWithStandardObjects(
dataSourceMetadata: DataSourceEntity,
workspaceId: string,
) {
const workspaceDataSource =
await this.tenantDataSourceService.connectToWorkspaceDataSource(
workspaceId,
);
if (!workspaceDataSource) {
throw new Error('Could not connect to workspace data source');
}
standardObjectsPrefillData(workspaceDataSource, dataSourceMetadata.schema);
}
/**
*
* Delete a workspace by deleting all metadata and the schema
*
* @param workspaceId
*/
public async delete(workspaceId: string): Promise<void> {
// Delete data from metadata tables
await this.fieldMetadataService.deleteFieldsMetadata(workspaceId);
await this.objectMetadataService.deleteObjectsMetadata(workspaceId);
await this.tenantMigrationService.delete(workspaceId);
await this.dataSourceMetadataService.delete(workspaceId);
// Delete schema
await this.tenantDataSourceService.deleteWorkspaceDBSchema(workspaceId);
}
}