admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: workspace health relation (#3466)

Browse Source 
feat: add check relation health
This commit is contained in:
Jérémy M
2024-01-17 17:05:35 +01:00
committed by GitHub
parent 64110c591a
commit 4b7e42c38e
9 changed files with 401 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/twenty-server/src/metadata/relation-metadata/relation-metadata.service.ts
View File

@ -26,6 +26,8 @@ import {
RelationMetadataType,
} from './relation-metadata.entity';
import { createRelationMetadataForeignKey } from './utils/create-relation-metadata-foreign-key.util';
@Injectable()
export class RelationMetadataService extends TypeOrmQueryService<RelationMetadataEntity> {
constructor(
@ -54,10 +56,10 @@ export class RelationMetadataService extends TypeOrmQueryService<RelationMetadat
// NOTE: this logic is called to create relation through metadata graphql endpoint (so only for custom field relations)
const isCustom = true;
const baseColumnName = `${camelCase(relationMetadataInput.toName)}Id`;
const foreignKeyColumnName = isCustom
? createCustomColumnName(baseColumnName)
: baseColumnName;
const foreignKeyColumnName = createRelationMetadataForeignKey(
relationMetadataInput.toName,
isCustom,
);
const createdFields = await this.fieldMetadataService.createMany([
this.createFieldMetadataForRelationMetadata(

15
packages/twenty-server/src/metadata/relation-metadata/utils/create-relation-metadata-foreign-key.util.ts Normal file
View File

@ -0,0 +1,15 @@
import { createCustomColumnName } from 'src/metadata/utils/create-custom-column-name.util';
import { camelCase } from 'src/utils/camel-case';
export const createRelationMetadataForeignKey = (
name: string,
isCustom?: boolean,
) => {
const baseColumnName = `${camelCase(name)}Id`;
const foreignKeyColumnName = isCustom
? createCustomColumnName(baseColumnName)
: baseColumnName;
return foreignKeyColumnName;
};

23
packages/twenty-server/src/workspace/workspace-health/interfaces/workspace-health-issue.interface.ts
View File

@ -2,6 +2,7 @@ import { WorkspaceTableStructure } from 'src/workspace/workspace-health/interfac
import { FieldMetadataEntity } from 'src/metadata/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { RelationMetadataEntity } from 'src/metadata/relation-metadata/relation-metadata.entity';
export enum WorkspaceHealthIssueType {
MISSING_TABLE = 'MISSING_TABLE',
@ -23,6 +24,11 @@ export enum WorkspaceHealthIssueType {
COLUMN_DEFAULT_VALUE_CONFLICT = 'COLUMN_DEFAULT_VALUE_CONFLICT',
COLUMN_DEFAULT_VALUE_NOT_VALID = 'COLUMN_DEFAULT_VALUE_NOT_VALID',
COLUMN_OPTIONS_NOT_VALID = 'COLUMN_OPTIONS_NOT_VALID',
RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID = 'RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID',
RELATION_FOREIGN_KEY_NOT_VALID = 'RELATION_FOREIGN_KEY_NOT_VALID',
RELATION_NULLABILITY_CONFLICT = 'RELATION_NULLABILITY_CONFLICT',
RELATION_FOREIGN_KEY_CONFLICT = 'RELATION_FOREIGN_KEY_CONFLICT',
RELATION_TYPE_NOT_VALID = 'RELATION_TYPE_NOT_VALID',
}
export interface WorkspaceHealthTableIssue {
@ -57,6 +63,21 @@ export interface WorkspaceHealthColumnIssue {
message: string;
}
export interface WorkspaceHealthRelationIssue {
type:
| WorkspaceHealthIssueType.RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID
| WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_NOT_VALID
| WorkspaceHealthIssueType.RELATION_NULLABILITY_CONFLICT
| WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_CONFLICT
| WorkspaceHealthIssueType.RELATION_TYPE_NOT_VALID;
fromFieldMetadata: FieldMetadataEntity | undefined;
toFieldMetadata: FieldMetadataEntity | undefined;
relationMetadata: RelationMetadataEntity;
columnStructure?: WorkspaceTableStructure;
message: string;
}
export type WorkspaceHealthIssue =
| WorkspaceHealthTableIssue
| WorkspaceHealthColumnIssue;
| WorkspaceHealthColumnIssue
| WorkspaceHealthRelationIssue;

10
packages/twenty-server/src/workspace/workspace-health/interfaces/workspace-table-definition.interface.ts
View File

@ -3,7 +3,13 @@ export interface WorkspaceTableStructure {
tableName: string;
columnName: string;
dataType: string;
isNullable: string;
columnDefault: string;
isPrimaryKey: string;
isNullable: boolean;
isPrimaryKey: boolean;
isForeignKey: boolean;
isUnique: boolean;
}
export type WorkspaceTableStructureResult = {
[P in keyof WorkspaceTableStructure]: string;
};

113
packages/twenty-server/src/workspace/workspace-health/services/database-structure.service.ts
View File

@ -3,7 +3,10 @@ import { Injectable } from '@nestjs/common';
import { ColumnType } from 'typeorm';
import { ColumnMetadata } from 'typeorm/metadata/ColumnMetadata';
import { WorkspaceTableStructure } from 'src/workspace/workspace-health/interfaces/workspace-table-definition.interface';
import {
WorkspaceTableStructure,
WorkspaceTableStructureResult,
} from 'src/workspace/workspace-health/interfaces/workspace-table-definition.interface';
import { FieldMetadataDefaultValue } from 'src/metadata/field-metadata/interfaces/field-metadata-default-value.interface';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
@ -20,33 +23,101 @@ export class DatabaseStructureService {
tableName: string,
): Promise<WorkspaceTableStructure[]> {
const mainDataSource = this.typeORMService.getMainDataSource();
return mainDataSource.query<WorkspaceTableStructure[]>(`
SELECT
c.table_schema as "tableSchema",
c.table_name as "tableName",
c.column_name as "columnName",
c.data_type as "dataType",
c.is_nullable as "isNullable",
c.column_default as "columnDefault",
CASE
const results = await mainDataSource.query<
WorkspaceTableStructureResult[]
>(`
WITH foreign_keys AS (
SELECT
kcu.table_schema AS schema_name,
kcu.table_name AS table_name,
kcu.column_name AS column_name,
tc.constraint_name AS constraint_name
FROM
information_schema.key_column_usage AS kcu
JOIN
information_schema.table_constraints AS tc
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE
tc.constraint_type = 'FOREIGN KEY'
AND tc.table_schema = '${schemaName}'
AND tc.table_name = '${tableName}'
),
unique_constraints AS (
SELECT
tc.table_schema AS schema_name,
tc.table_name AS table_name,
kcu.column_name AS column_name
FROM
information_schema.key_column_usage AS kcu
JOIN
information_schema.table_constraints AS tc
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE
tc.constraint_type = 'UNIQUE'
AND tc.table_schema = '${schemaName}'
AND tc.table_name = '${tableName}'
)
SELECT
c.table_schema AS "tableSchema",
c.table_name AS "tableName",
c.column_name AS "columnName",
c.data_type AS "dataType",
c.is_nullable AS "isNullable",
c.column_default AS "columnDefault",
CASE
WHEN pk.constraint_type = 'PRIMARY KEY' THEN 'TRUE'
ELSE 'FALSE'
END as "isPrimaryKey"
FROM
information_schema.columns c
LEFT JOIN
information_schema.constraint_column_usage as ccu ON c.column_name = ccu.column_name AND c.table_name = ccu.table_name
LEFT JOIN
information_schema.table_constraints as pk ON pk.constraint_name = ccu.constraint_name AND pk.constraint_type = 'PRIMARY KEY'
WHERE
END AS "isPrimaryKey",
CASE
WHEN fk.constraint_name IS NOT NULL THEN 'TRUE'
ELSE 'FALSE'
END AS "isForeignKey",
CASE
WHEN uc.column_name IS NOT NULL THEN 'TRUE'
ELSE 'FALSE'
END AS "isUnique"
FROM
information_schema.columns AS c
LEFT JOIN
information_schema.constraint_column_usage AS ccu
ON c.column_name = ccu.column_name
AND c.table_name = ccu.table_name
LEFT JOIN
information_schema.table_constraints AS pk
ON pk.constraint_name = ccu.constraint_name
AND pk.constraint_type = 'PRIMARY KEY'
LEFT JOIN
foreign_keys AS fk
ON c.table_schema = fk.schema_name
AND c.table_name = fk.table_name
AND c.column_name = fk.column_name
LEFT JOIN
unique_constraints AS uc
ON c.table_schema = uc.schema_name
AND c.table_name = uc.table_name
AND c.column_name = uc.column_name
WHERE
c.table_schema = '${schemaName}'
AND c.table_name = '${tableName}';
`);
if (!results || results.length === 0) {
return [];
}
return results.map((item) => ({
...item,
isNullable: item.isNullable === 'YES',
isPrimaryKey: item.isPrimaryKey === 'TRUE',
isForeignKey: item.isForeignKey === 'TRUE',
isUnique: item.isUnique === 'TRUE',
}));
}
getPostgresDataType(fieldMedataType: FieldMetadataType): string {
const typeORMType = fieldMetadataTypeToColumnType(fieldMedataType);
getPostgresDataType(fieldMetadataType: FieldMetadataType): string {
const typeORMType = fieldMetadataTypeToColumnType(fieldMetadataType);
const mainDataSource = this.typeORMService.getMainDataSource();
return mainDataSource.driver.normalizeType({

27
packages/twenty-server/src/workspace/workspace-health/services/field-metadata-health.service.ts
View File

@ -1,8 +1,4 @@
import {
Injectable,
InternalServerErrorException,
NotFoundException,
} from '@nestjs/common';
import { Injectable, InternalServerErrorException } from '@nestjs/common';
import {
WorkspaceHealthIssue,
@ -30,31 +26,19 @@ export class FieldMetadataHealthService {
) {}
async healthCheck(
schemaName: string,
tableName: string,
workspaceTableColumns: WorkspaceTableStructure[],
fieldMetadataCollection: FieldMetadataEntity[],
options: WorkspaceHealthOptions,
): Promise<WorkspaceHealthIssue[]> {
const issues: WorkspaceHealthIssue[] = [];
const workspaceTableColumns =
await this.databaseStructureService.getWorkspaceTableColumns(
schemaName,
tableName,
);
if (!workspaceTableColumns) {
throw new NotFoundException(
`Table ${tableName} not found in schema ${schemaName}`,
);
}
for (const fieldMetadata of fieldMetadataCollection) {
// Ignore relation fields for now
// Relation metadata are checked in another service
if (
fieldMetadata.fromRelationMetadata ||
fieldMetadata.toRelationMetadata
) {
// TODO: Check relation fields
continue;
}
@ -140,7 +124,6 @@ export class FieldMetadataHealthService {
fieldMetadata.type,
fieldMetadata.defaultValue,
);
const isNullable = fieldMetadata.isNullable ? 'TRUE' : 'FALSE';
// Check if column exist in database
const columnStructure = workspaceTableColumns.find(
(tableDefinition) => tableDefinition.columnName === columnName,
@ -167,7 +150,7 @@ export class FieldMetadataHealthService {
});
}
if (columnStructure.isNullable !== isNullable) {
if (columnStructure.isNullable !== fieldMetadata.isNullable) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_NULLABILITY_CONFLICT,
fieldMetadata,
@ -211,7 +194,7 @@ export class FieldMetadataHealthService {
issues.push(...targetColumnMapIssues);
if (fieldMetadata.isCustom && !columnName.startsWith('_')) {
if (fieldMetadata.isCustom && !columnName?.startsWith('_')) {
issues.push({
type: WorkspaceHealthIssueType.COLUMN_NAME_SHOULD_BE_CUSTOM,
fieldMetadata,

222
packages/twenty-server/src/workspace/workspace-health/services/relation-metadata.health.service.ts Normal file
View File

@ -0,0 +1,222 @@
import { Injectable } from '@nestjs/common';
import { WorkspaceTableStructure } from 'src/workspace/workspace-health/interfaces/workspace-table-definition.interface';
import {
WorkspaceHealthIssue,
WorkspaceHealthIssueType,
} from 'src/workspace/workspace-health/interfaces/workspace-health-issue.interface';
import {
WorkspaceHealthMode,
WorkspaceHealthOptions,
} from 'src/workspace/workspace-health/interfaces/workspace-health-options.interface';
import {
FieldMetadataEntity,
FieldMetadataType,
} from 'src/metadata/field-metadata/field-metadata.entity';
import {
RelationMetadataEntity,
RelationMetadataType,
} from 'src/metadata/relation-metadata/relation-metadata.entity';
import { createRelationMetadataForeignKey } from 'src/metadata/relation-metadata/utils/create-relation-metadata-foreign-key.util';
import {
RelationDirection,
deduceRelationDirection,
} from 'src/workspace/utils/deduce-relation-direction.util';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
@Injectable()
export class RelationMetadataHealthService {
constructor() {}
public healthCheck(
workspaceTableColumns: WorkspaceTableStructure[],
objectMetadataCollection: ObjectMetadataEntity[],
objectMetadata: ObjectMetadataEntity,
options: WorkspaceHealthOptions,
) {
const issues: WorkspaceHealthIssue[] = [];
for (const fieldMetadata of objectMetadata.fields) {
// We're only interested in relation fields
if (fieldMetadata.type !== FieldMetadataType.RELATION) {
continue;
}
const relationMetadata =
fieldMetadata.fromRelationMetadata ?? fieldMetadata.toRelationMetadata;
const relationDirection = deduceRelationDirection(
objectMetadata.id,
relationMetadata,
);
// Many to many relations are not supported yet
if (relationMetadata.relationType === RelationMetadataType.MANY_TO_MANY) {
return [];
}
const fromObjectMetadata = objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.fromObjectMetadataId,
);
const fromFieldMetadata = fromObjectMetadata?.fields.find(
(fieldMetadata) =>
fieldMetadata.id === relationMetadata.fromFieldMetadataId,
);
const toObjectMetadata = objectMetadataCollection.find(
(objectMetadata) =>
objectMetadata.id === relationMetadata.toObjectMetadataId,
);
const toFieldMetadata = toObjectMetadata?.fields.find(
(fieldMetadata) =>
fieldMetadata.id === relationMetadata.toFieldMetadataId,
);
if (!fromFieldMetadata || !toFieldMetadata) {
issues.push({
type: WorkspaceHealthIssueType.RELATION_FROM_OR_TO_FIELD_METADATA_NOT_VALID,
fromFieldMetadata,
toFieldMetadata,
relationMetadata,
message: `Relation ${relationMetadata.id} has invalid from or to field metadata`,
});
return issues;
}
if (
options.mode === WorkspaceHealthMode.All ||
options.mode === WorkspaceHealthMode.Structure
) {
// Check relation structure
const structureIssues = this.structureRelationCheck(
fromFieldMetadata,
toFieldMetadata,
toObjectMetadata?.fields ?? [],
relationDirection,
relationMetadata,
workspaceTableColumns,
);
issues.push(...structureIssues);
}
if (
options.mode === WorkspaceHealthMode.All ||
options.mode === WorkspaceHealthMode.Metadata
) {
// Check relation metadata
const metadataIssues = this.metadataRelationCheck(
fromFieldMetadata,
toFieldMetadata,
relationDirection,
relationMetadata,
);
issues.push(...metadataIssues);
}
}
return issues;
}
private structureRelationCheck(
fromFieldMetadata: FieldMetadataEntity,
toFieldMetadata: FieldMetadataEntity,
toObjectMetadataFields: FieldMetadataEntity[],
relationDirection: RelationDirection,
relationMetadata: RelationMetadataEntity,
workspaceTableColumns: WorkspaceTableStructure[],
): WorkspaceHealthIssue[] {
const issues: WorkspaceHealthIssue[] = [];
// Nothing to check on the structure
if (relationDirection === RelationDirection.FROM) {
return [];
}
const isCustom = toFieldMetadata.isCustom ?? false;
const foreignKeyColumnName = createRelationMetadataForeignKey(
toFieldMetadata.name,
isCustom,
);
const relationColumn = workspaceTableColumns.find(
(column) => column.columnName === foreignKeyColumnName,
);
const relationFieldMetadata = toObjectMetadataFields.find(
(fieldMetadata) => fieldMetadata.name === foreignKeyColumnName,
);
if (!relationColumn || !relationFieldMetadata) {
issues.push({
type: WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_NOT_VALID,
fromFieldMetadata,
toFieldMetadata,
relationMetadata,
message: `Relation ${relationMetadata.id} doesn't have a valid foreign key`,
});
return issues;
}
if (!relationColumn.isForeignKey) {
issues.push({
type: WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_CONFLICT,
fromFieldMetadata,
toFieldMetadata,
relationMetadata,
message: `Relation ${relationMetadata.id} foreign key is not properly set`,
});
}
if (relationColumn.isNullable !== relationFieldMetadata.isNullable) {
issues.push({
type: WorkspaceHealthIssueType.RELATION_NULLABILITY_CONFLICT,
fromFieldMetadata,
toFieldMetadata,
relationMetadata,
message: `Relation ${relationMetadata.id} foreign key is not properly set`,
});
}
if (
relationMetadata.relationType === RelationMetadataType.ONE_TO_ONE &&
!relationColumn.isUnique
) {
issues.push({
type: WorkspaceHealthIssueType.RELATION_FOREIGN_KEY_CONFLICT,
fromFieldMetadata,
toFieldMetadata,
relationMetadata,
message: `Relation ${relationMetadata.id} foreign key is not marked as unique and relation type is one-to-one`,
});
}
return issues;
}
private metadataRelationCheck(
fromFieldMetadata: FieldMetadataEntity,
toFieldMetadata: FieldMetadataEntity,
relationDirection: RelationDirection,
relationMetadata: RelationMetadataEntity,
): WorkspaceHealthIssue[] {
const issues: WorkspaceHealthIssue[] = [];
if (
!Object.values(RelationMetadataType).includes(
relationMetadata.relationType,
)
) {
issues.push({
type: WorkspaceHealthIssueType.RELATION_TYPE_NOT_VALID,
fromFieldMetadata,
toFieldMetadata,
relationMetadata,
message: `Relation ${relationMetadata.id} has invalid relation type`,
});
}
return issues;
}
}

2
packages/twenty-server/src/workspace/workspace-health/workspace-health.module.ts
View File

@ -7,6 +7,7 @@ import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/wo
import { DatabaseStructureService } from 'src/workspace/workspace-health/services/database-structure.service';
import { FieldMetadataHealthService } from 'src/workspace/workspace-health/services/field-metadata-health.service';
import { ObjectMetadataHealthService } from 'src/workspace/workspace-health/services/object-metadata-health.service';
import { RelationMetadataHealthService } from 'src/workspace/workspace-health/services/relation-metadata.health.service';
import { WorkspaceHealthService } from 'src/workspace/workspace-health/workspace-health.service';
@Module({
@ -21,6 +22,7 @@ import { WorkspaceHealthService } from 'src/workspace/workspace-health/workspace
DatabaseStructureService,
ObjectMetadataHealthService,
FieldMetadataHealthService,
RelationMetadataHealthService,
],
exports: [WorkspaceHealthService],
})

31
packages/twenty-server/src/workspace/workspace-health/workspace-health.service.ts
View File

@ -12,6 +12,8 @@ import { ObjectMetadataService } from 'src/metadata/object-metadata/object-metad
import { WorkspaceDataSourceService } from 'src/workspace/workspace-datasource/workspace-datasource.service';
import { ObjectMetadataHealthService } from 'src/workspace/workspace-health/services/object-metadata-health.service';
import { FieldMetadataHealthService } from 'src/workspace/workspace-health/services/field-metadata-health.service';
import { RelationMetadataHealthService } from 'src/workspace/workspace-health/services/relation-metadata.health.service';
import { DatabaseStructureService } from 'src/workspace/workspace-health/services/database-structure.service';
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
@Injectable()
@ -20,9 +22,11 @@ export class WorkspaceHealthService {
private readonly dataSourceService: DataSourceService,
private readonly typeORMService: TypeORMService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly databaseStructureService: DatabaseStructureService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly objectMetadataHealthService: ObjectMetadataHealthService,
private readonly fieldMetadataHealthService: FieldMetadataHealthService,
private readonly relationMetadataHealthService: RelationMetadataHealthService,
) {}
async healthCheck(
@ -41,7 +45,7 @@ export class WorkspaceHealthService {
// Check if a data source exists for this workspace
if (!dataSourceMetadata) {
throw new NotFoundException(
`Datasource for workspace id ${workspaceId} not found`,
`DataSource for workspace id ${workspaceId} not found`,
);
}
@ -57,6 +61,19 @@ export class WorkspaceHealthService {
}
for (const objectMetadata of objectMetadataCollection) {
const tableName = computeObjectTargetTable(objectMetadata);
const workspaceTableColumns =
await this.databaseStructureService.getWorkspaceTableColumns(
schemaName,
tableName,
);
if (!workspaceTableColumns || workspaceTableColumns.length === 0) {
throw new NotFoundException(
`Table ${tableName} not found in schema ${schemaName}`,
);
}
// Check object metadata health
const objectIssues = await this.objectMetadataHealthService.healthCheck(
schemaName,
@ -68,13 +85,23 @@ export class WorkspaceHealthService {
// Check fields metadata health
const fieldIssues = await this.fieldMetadataHealthService.healthCheck(
schemaName,
computeObjectTargetTable(objectMetadata),
workspaceTableColumns,
objectMetadata.fields,
options,
);
issues.push(...fieldIssues);
// Check relation metadata health
const relationIssues = this.relationMetadataHealthService.healthCheck(
workspaceTableColumns,
objectMetadataCollection,
objectMetadata,
options,
);
issues.push(...relationIssues);
}
return issues;