feat: workspace sync (#3505)
* feat: wip workspace sync * feat: wip lot of debugging * feat: refactor and fix sync * fix: clean fix: clean * feat: add simple comparator tests * fix: remove debug * feat: wip drop table * fix: main merge * fix: some issues, and prepare storage system to handle complex deletion * feat: wip clean and fix * fix: reflect issue when using array instead of map and clean * fix: test & sync * fix: yarn files * fix: unecesary if-else * fix: if condition not needed * fix: remove debug * fix: replace EQUAL by SKIP * fix: sync metadata relation not applied properly * fix: lint issues * fix: merge issue
This commit is contained in:
Jérémy M
@ -0,0 +1,34 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
|
||||
|
||||
import { FeatureFlagEntity } from 'src/core/feature-flag/feature-flag.entity';
|
||||
|
||||
@Injectable()
|
||||
export class FeatureFlagFactory {
|
||||
constructor(
|
||||
@InjectRepository(FeatureFlagEntity, 'core')
|
||||
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
|
||||
) {}
|
||||
|
||||
async create(context: WorkspaceSyncContext): Promise<FeatureFlagMap> {
|
||||
const workspaceFeatureFlags = await this.featureFlagRepository.find({
|
||||
where: { workspaceId: context.workspaceId },
|
||||
});
|
||||
|
||||
const workspaceFeatureFlagsMap = workspaceFeatureFlags.reduce(
|
||||
(result, currentFeatureFlag) => {
|
||||
result[currentFeatureFlag.key] = currentFeatureFlag.value;
|
||||
|
||||
return result;
|
||||
},
|
||||
{} as FeatureFlagMap,
|
||||
);
|
||||
|
||||
return workspaceFeatureFlagsMap;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
import { FeatureFlagFactory } from './feature-flags.factory';
|
||||
import { StandardObjectFactory } from './standard-object.factory';
|
||||
import { StandardRelationFactory } from './standard-relation.factory';
|
||||
import { WorkspaceSyncFactory } from './workspace-sync.factory';
|
||||
|
||||
export const workspaceSyncMetadataFactories = [
|
||||
FeatureFlagFactory,
|
||||
StandardObjectFactory,
|
||||
StandardRelationFactory,
|
||||
WorkspaceSyncFactory,
|
||||
];
|
||||
@ -0,0 +1,70 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { PartialObjectMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-object-metadata.interface';
|
||||
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
|
||||
import { PartialFieldMetadata } from 'src/workspace/workspace-sync-metadata/interfaces/partial-field-metadata.interface';
|
||||
|
||||
import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
|
||||
import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects';
|
||||
import { TypedReflect } from 'src/utils/typed-reflect';
|
||||
import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util';
|
||||
|
||||
@Injectable()
|
||||
export class StandardObjectFactory {
|
||||
create(
|
||||
context: WorkspaceSyncContext,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): PartialObjectMetadata[] {
|
||||
return standardObjectMetadata
|
||||
.map((metadata) =>
|
||||
this.createObjectMetadata(metadata, context, workspaceFeatureFlagsMap),
|
||||
)
|
||||
.filter((metadata): metadata is PartialObjectMetadata => !!metadata);
|
||||
}
|
||||
|
||||
private createObjectMetadata(
|
||||
metadata: typeof BaseObjectMetadata,
|
||||
context: WorkspaceSyncContext,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): PartialObjectMetadata | undefined {
|
||||
const objectMetadata = TypedReflect.getMetadata('objectMetadata', metadata);
|
||||
const fieldMetadataMap =
|
||||
TypedReflect.getMetadata('fieldMetadataMap', metadata) ?? [];
|
||||
|
||||
if (!objectMetadata) {
|
||||
throw new Error(
|
||||
`Object metadata decorator not found, can\'t parse ${metadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (isGatedAndNotEnabled(objectMetadata.gate, workspaceFeatureFlagsMap)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const fields = Object.values(fieldMetadataMap).reduce(
|
||||
// Omit gate as we don't want to store it in the DB
|
||||
(acc, { gate, ...fieldMetadata }) => {
|
||||
if (isGatedAndNotEnabled(gate, workspaceFeatureFlagsMap)) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
acc.push({
|
||||
...fieldMetadata,
|
||||
workspaceId: context.workspaceId,
|
||||
isSystem: objectMetadata.isSystem || fieldMetadata.isSystem,
|
||||
});
|
||||
|
||||
return acc;
|
||||
},
|
||||
[] as PartialFieldMetadata[],
|
||||
);
|
||||
|
||||
return {
|
||||
...objectMetadata,
|
||||
workspaceId: context.workspaceId,
|
||||
dataSourceId: context.dataSourceId,
|
||||
fields,
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,116 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { WorkspaceSyncContext } from 'src/workspace/workspace-sync-metadata/interfaces/workspace-sync-context.interface';
|
||||
import { FeatureFlagMap } from 'src/core/feature-flag/interfaces/feature-flag-map.interface';
|
||||
|
||||
import { BaseObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects/base.object-metadata';
|
||||
import { standardObjectMetadata } from 'src/workspace/workspace-sync-metadata/standard-objects';
|
||||
import { TypedReflect } from 'src/utils/typed-reflect';
|
||||
import { isGatedAndNotEnabled } from 'src/workspace/workspace-sync-metadata/utils/is-gate-and-not-enabled.util';
|
||||
import { assert } from 'src/utils/assert';
|
||||
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
|
||||
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
|
||||
|
||||
@Injectable()
|
||||
export class StandardRelationFactory {
|
||||
create(
|
||||
context: WorkspaceSyncContext,
|
||||
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): Partial<RelationMetadataEntity>[] {
|
||||
return standardObjectMetadata.flatMap((standardObjectMetadata) =>
|
||||
this.createRelationMetadata(
|
||||
standardObjectMetadata,
|
||||
context,
|
||||
originalObjectMetadataMap,
|
||||
workspaceFeatureFlagsMap,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private createRelationMetadata(
|
||||
standardObjectMetadata: typeof BaseObjectMetadata,
|
||||
context: WorkspaceSyncContext,
|
||||
originalObjectMetadataMap: Record<string, ObjectMetadataEntity>,
|
||||
workspaceFeatureFlagsMap: FeatureFlagMap,
|
||||
): Partial<RelationMetadataEntity>[] {
|
||||
const objectMetadata = TypedReflect.getMetadata(
|
||||
'objectMetadata',
|
||||
standardObjectMetadata,
|
||||
);
|
||||
const relationMetadataCollection = TypedReflect.getMetadata(
|
||||
'relationMetadataCollection',
|
||||
standardObjectMetadata,
|
||||
);
|
||||
|
||||
if (!objectMetadata) {
|
||||
throw new Error(
|
||||
`Object metadata decorator not found, can\'t parse ${standardObjectMetadata.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
!relationMetadataCollection ||
|
||||
isGatedAndNotEnabled(objectMetadata.gate, workspaceFeatureFlagsMap)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return relationMetadataCollection
|
||||
.filter(
|
||||
(relationMetadata) =>
|
||||
!isGatedAndNotEnabled(
|
||||
relationMetadata.gate,
|
||||
workspaceFeatureFlagsMap,
|
||||
),
|
||||
)
|
||||
.map((relationMetadata) => {
|
||||
const fromObjectMetadata =
|
||||
originalObjectMetadataMap[relationMetadata.fromObjectNameSingular];
|
||||
|
||||
assert(
|
||||
fromObjectMetadata,
|
||||
`Object ${relationMetadata.fromObjectNameSingular} not found in DB
|
||||
for relation FROM defined in class ${objectMetadata.nameSingular}`,
|
||||
);
|
||||
|
||||
const toObjectMetadata =
|
||||
originalObjectMetadataMap[relationMetadata.toObjectNameSingular];
|
||||
|
||||
assert(
|
||||
toObjectMetadata,
|
||||
`Object ${relationMetadata.toObjectNameSingular} not found in DB
|
||||
for relation TO defined in class ${objectMetadata.nameSingular}`,
|
||||
);
|
||||
|
||||
const fromFieldMetadata = fromObjectMetadata?.fields.find(
|
||||
(field) => field.name === relationMetadata.fromFieldMetadataName,
|
||||
);
|
||||
|
||||
assert(
|
||||
fromFieldMetadata,
|
||||
`Field ${relationMetadata.fromFieldMetadataName} not found in object ${relationMetadata.fromObjectNameSingular}
|
||||
for relation FROM defined in class ${objectMetadata.nameSingular}`,
|
||||
);
|
||||
|
||||
const toFieldMetadata = toObjectMetadata?.fields.find(
|
||||
(field) => field.name === relationMetadata.toFieldMetadataName,
|
||||
);
|
||||
|
||||
assert(
|
||||
toFieldMetadata,
|
||||
`Field ${relationMetadata.toFieldMetadataName} not found in object ${relationMetadata.toObjectNameSingular}
|
||||
for relation TO defined in class ${objectMetadata.nameSingular}`,
|
||||
);
|
||||
|
||||
return {
|
||||
relationType: relationMetadata.type,
|
||||
fromObjectMetadataId: fromObjectMetadata?.id,
|
||||
toObjectMetadataId: toObjectMetadata?.id,
|
||||
fromFieldMetadataId: fromFieldMetadata?.id,
|
||||
toFieldMetadataId: toFieldMetadata?.id,
|
||||
workspaceId: context.workspaceId,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,264 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
FieldMetadataEntity,
|
||||
FieldMetadataType,
|
||||
} from 'src/metadata/field-metadata/field-metadata.entity';
|
||||
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
|
||||
import {
|
||||
WorkspaceMigrationColumnActionType,
|
||||
WorkspaceMigrationColumnRelation,
|
||||
WorkspaceMigrationEntity,
|
||||
WorkspaceMigrationTableAction,
|
||||
} from 'src/metadata/workspace-migration/workspace-migration.entity';
|
||||
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
|
||||
import { WorkspaceMigrationFactory } from 'src/metadata/workspace-migration/workspace-migration.factory';
|
||||
import {
|
||||
RelationMetadataEntity,
|
||||
RelationMetadataType,
|
||||
} from 'src/metadata/relation-metadata/relation-metadata.entity';
|
||||
import { camelCase } from 'src/utils/camel-case';
|
||||
import { generateMigrationName } from 'src/metadata/workspace-migration/utils/generate-migration-name.util';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceSyncFactory {
|
||||
constructor(
|
||||
private readonly workspaceMigrationFactory: WorkspaceMigrationFactory,
|
||||
) {}
|
||||
|
||||
async createObjectMigration(
|
||||
originalObjectMetadataCollection: ObjectMetadataEntity[],
|
||||
createdObjectMetadataCollection: ObjectMetadataEntity[],
|
||||
objectMetadataDeleteCollection: ObjectMetadataEntity[],
|
||||
createdFieldMetadataCollection: FieldMetadataEntity[],
|
||||
fieldMetadataDeleteCollection: FieldMetadataEntity[],
|
||||
): Promise<Partial<WorkspaceMigrationEntity>[]> {
|
||||
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
|
||||
|
||||
/**
|
||||
* Create object migrations
|
||||
*/
|
||||
if (createdObjectMetadataCollection.length > 0) {
|
||||
for (const objectMetadata of createdObjectMetadataCollection) {
|
||||
const migrations = [
|
||||
{
|
||||
name: computeObjectTargetTable(objectMetadata),
|
||||
action: 'create',
|
||||
} satisfies WorkspaceMigrationTableAction,
|
||||
...objectMetadata.fields
|
||||
.filter((field) => field.type !== FieldMetadataType.RELATION)
|
||||
.map(
|
||||
(field) =>
|
||||
({
|
||||
name: computeObjectTargetTable(objectMetadata),
|
||||
action: 'alter',
|
||||
columns: this.workspaceMigrationFactory.createColumnActions(
|
||||
WorkspaceMigrationColumnActionType.CREATE,
|
||||
field,
|
||||
),
|
||||
}) satisfies WorkspaceMigrationTableAction,
|
||||
),
|
||||
];
|
||||
|
||||
workspaceMigrations.push({
|
||||
workspaceId: objectMetadata.workspaceId,
|
||||
name: generateMigrationName(`create-${objectMetadata.nameSingular}`),
|
||||
isCustom: false,
|
||||
migrations,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete object migrations
|
||||
* TODO: handle object delete migrations.
|
||||
* Note: we need to delete the relation first due to the DB constraint.
|
||||
*/
|
||||
// if (objectMetadataDeleteCollection.length > 0) {
|
||||
// for (const objectMetadata of objectMetadataDeleteCollection) {
|
||||
// const migrations = [
|
||||
// {
|
||||
// name: computeObjectTargetTable(objectMetadata),
|
||||
// action: 'drop',
|
||||
// columns: [],
|
||||
// } satisfies WorkspaceMigrationTableAction,
|
||||
// ];
|
||||
|
||||
// workspaceMigrations.push({
|
||||
// workspaceId: objectMetadata.workspaceId,
|
||||
// isCustom: false,
|
||||
// migrations,
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
|
||||
/**
|
||||
* Create field migrations
|
||||
*/
|
||||
const originalObjectMetadataMap = originalObjectMetadataCollection.reduce(
|
||||
(result, currentObject) => {
|
||||
result[currentObject.id] = currentObject;
|
||||
|
||||
return result;
|
||||
},
|
||||
{} as Record<string, ObjectMetadataEntity>,
|
||||
);
|
||||
|
||||
if (createdFieldMetadataCollection.length > 0) {
|
||||
for (const fieldMetadata of createdFieldMetadataCollection) {
|
||||
const migrations = [
|
||||
{
|
||||
name: computeObjectTargetTable(
|
||||
originalObjectMetadataMap[fieldMetadata.objectMetadataId],
|
||||
),
|
||||
action: 'alter',
|
||||
columns: this.workspaceMigrationFactory.createColumnActions(
|
||||
WorkspaceMigrationColumnActionType.CREATE,
|
||||
fieldMetadata,
|
||||
),
|
||||
} satisfies WorkspaceMigrationTableAction,
|
||||
];
|
||||
|
||||
workspaceMigrations.push({
|
||||
workspaceId: fieldMetadata.workspaceId,
|
||||
name: generateMigrationName(`create-${fieldMetadata.name}`),
|
||||
isCustom: false,
|
||||
migrations,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete field migrations
|
||||
*/
|
||||
if (fieldMetadataDeleteCollection.length > 0) {
|
||||
for (const fieldMetadata of fieldMetadataDeleteCollection) {
|
||||
const migrations = [
|
||||
{
|
||||
name: computeObjectTargetTable(
|
||||
originalObjectMetadataMap[fieldMetadata.objectMetadataId],
|
||||
),
|
||||
action: 'alter',
|
||||
columns: [
|
||||
{
|
||||
action: WorkspaceMigrationColumnActionType.DROP,
|
||||
columnName: fieldMetadata.name,
|
||||
},
|
||||
],
|
||||
} satisfies WorkspaceMigrationTableAction,
|
||||
];
|
||||
|
||||
workspaceMigrations.push({
|
||||
workspaceId: fieldMetadata.workspaceId,
|
||||
name: generateMigrationName(`delete-${fieldMetadata.name}`),
|
||||
isCustom: false,
|
||||
migrations,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return workspaceMigrations;
|
||||
}
|
||||
|
||||
async createRelationMigration(
|
||||
originalObjectMetadataCollection: ObjectMetadataEntity[],
|
||||
createdRelationMetadataCollection: RelationMetadataEntity[],
|
||||
// TODO: handle relation deletion
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
relationMetadataDeleteCollection: RelationMetadataEntity[],
|
||||
): Promise<Partial<WorkspaceMigrationEntity>[]> {
|
||||
const workspaceMigrations: Partial<WorkspaceMigrationEntity>[] = [];
|
||||
|
||||
if (createdRelationMetadataCollection.length > 0) {
|
||||
for (const relationMetadata of createdRelationMetadataCollection) {
|
||||
const toObjectMetadata = originalObjectMetadataCollection.find(
|
||||
(object) => object.id === relationMetadata.toObjectMetadataId,
|
||||
);
|
||||
|
||||
const fromObjectMetadata = originalObjectMetadataCollection.find(
|
||||
(object) => object.id === relationMetadata.fromObjectMetadataId,
|
||||
);
|
||||
|
||||
if (!toObjectMetadata) {
|
||||
throw new Error(
|
||||
`ObjectMetadata with id ${relationMetadata.toObjectMetadataId} not found`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!fromObjectMetadata) {
|
||||
throw new Error(
|
||||
`ObjectMetadata with id ${relationMetadata.fromObjectMetadataId} not found`,
|
||||
);
|
||||
}
|
||||
|
||||
const toFieldMetadata = toObjectMetadata.fields.find(
|
||||
(field) => field.id === relationMetadata.toFieldMetadataId,
|
||||
);
|
||||
|
||||
if (!toFieldMetadata) {
|
||||
throw new Error(
|
||||
`FieldMetadata with id ${relationMetadata.toFieldMetadataId} not found`,
|
||||
);
|
||||
}
|
||||
|
||||
const migrations = [
|
||||
{
|
||||
name: computeObjectTargetTable(toObjectMetadata),
|
||||
action: 'alter',
|
||||
columns: [
|
||||
{
|
||||
action: WorkspaceMigrationColumnActionType.RELATION,
|
||||
columnName: `${camelCase(toFieldMetadata.name)}Id`,
|
||||
referencedTableName:
|
||||
computeObjectTargetTable(fromObjectMetadata),
|
||||
referencedTableColumnName: 'id',
|
||||
isUnique:
|
||||
relationMetadata.relationType ===
|
||||
RelationMetadataType.ONE_TO_ONE,
|
||||
} satisfies WorkspaceMigrationColumnRelation,
|
||||
],
|
||||
} satisfies WorkspaceMigrationTableAction,
|
||||
];
|
||||
|
||||
workspaceMigrations.push({
|
||||
workspaceId: relationMetadata.workspaceId,
|
||||
name: generateMigrationName(
|
||||
`create-relation-from-${fromObjectMetadata.nameSingular}-to-${toObjectMetadata.nameSingular}`,
|
||||
),
|
||||
isCustom: false,
|
||||
migrations,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// if (relationMetadataDeleteCollection.length > 0) {
|
||||
// for (const relationMetadata of relationMetadataDeleteCollection) {
|
||||
// const toObjectMetadata = originalObjectMetadataCollection.find(
|
||||
// (object) => object.id === relationMetadata.toObjectMetadataId,
|
||||
// );
|
||||
|
||||
// if (!toObjectMetadata) {
|
||||
// throw new Error(
|
||||
// `ObjectMetadata with id ${relationMetadata.toObjectMetadataId} not found`,
|
||||
// );
|
||||
// }
|
||||
|
||||
// const migrations = [
|
||||
// {
|
||||
// name: computeObjectTargetTable(toObjectMetadata),
|
||||
// action: 'drop',
|
||||
// columns: [],
|
||||
// } satisfies WorkspaceMigrationTableAction,
|
||||
// ];
|
||||
|
||||
// workspaceMigrations.push({
|
||||
// workspaceId: relationMetadata.workspaceId,
|
||||
// isCustom: false,
|
||||
// migrations,
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
|
||||
return workspaceMigrations;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user