Enforce unique constraints for 0.33 (#8645)

## Context
- Fixing folder structure where 0.33 was inside 0.32.
- Updating the command so it runs enforce uniqueness on all tables
- Updating workspaces entities so the sync metadata adds the index
This commit is contained in:
Weiko
2024-11-21 17:39:26 +01:00
committed by GitHub
parent 39373b4a28
commit 52df5301a8
13 changed files with 130 additions and 86 deletions

View File

@ -8,7 +8,7 @@ import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-dem
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
import { UpgradeTo0_32CommandModule } from 'src/database/commands/upgrade-version/0-32/0-32-upgrade-version.module';
import { UpgradeTo0_33CommandModule } from 'src/database/commands/upgrade-version/0-32/0-33/0-33-upgrade-version.module';
import { UpgradeTo0_33CommandModule } from 'src/database/commands/upgrade-version/0-33/0-33-upgrade-version.module';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { BillingSubscription } from 'src/engine/core-modules/billing/entities/billing-subscription.entity';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';

View File

@ -10,8 +10,6 @@ import { SimplifySearchVectorExpressionCommand } from 'src/database/commands/upg
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
import { EnforceUniqueConstraintsCommand } from './0-32-enforce-unique-constraints.command';
interface UpdateTo0_32CommandOptions {
workspaceId?: string;
}
@ -25,7 +23,6 @@ export class UpgradeTo0_32Command extends ActiveWorkspacesCommandRunner {
@InjectRepository(Workspace, 'core')
protected readonly workspaceRepository: Repository<Workspace>,
private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
private readonly enforceUniqueConstraintsCommand: EnforceUniqueConstraintsCommand,
private readonly simplifySearchVectorExpressionCommand: SimplifySearchVectorExpressionCommand,
private readonly copyWebhookOperationIntoOperationsCommand: CopyWebhookOperationIntoOperationsCommand,
private readonly backfillViewGroupsCommand: BackfillViewGroupsCommand,
@ -53,12 +50,6 @@ export class UpgradeTo0_32Command extends ActiveWorkspacesCommandRunner {
workspaceIds,
);
await this.enforceUniqueConstraintsCommand.executeActiveWorkspacesCommand(
passedParam,
options,
workspaceIds,
);
await this.copyWebhookOperationIntoOperationsCommand.executeActiveWorkspacesCommand(
passedParam,
options,

View File

@ -2,7 +2,6 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { CopyWebhookOperationIntoOperationsCommand } from 'src/database/commands/upgrade-version/0-32/0-32-copy-webhook-operation-into-operations-command';
import { EnforceUniqueConstraintsCommand } from 'src/database/commands/upgrade-version/0-32/0-32-enforce-unique-constraints.command';
import { SimplifySearchVectorExpressionCommand } from 'src/database/commands/upgrade-version/0-32/0-32-simplify-search-vector-expression';
import { UpgradeTo0_32Command } from 'src/database/commands/upgrade-version/0-32/0-32-upgrade-version.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
@ -27,7 +26,6 @@ import { BackfillViewGroupsCommand } from './0-32-backfill-view-groups.command';
],
providers: [
UpgradeTo0_32Command,
EnforceUniqueConstraintsCommand,
BackfillViewGroupsCommand,
CopyWebhookOperationIntoOperationsCommand,
SimplifySearchVectorExpressionCommand,

View File

@ -17,10 +17,11 @@ interface EnforceUniqueConstraintsCommandOptions
company?: boolean;
viewField?: boolean;
viewSort?: boolean;
verbose?: boolean;
}
@Command({
name: 'upgrade-0.32:enforce-unique-constraints',
name: 'upgrade-0.33:enforce-unique-constraints',
description:
'Enforce unique constraints on company domainName, person emailsPrimaryEmail, ViewField, and ViewSort',
})
@ -41,6 +42,14 @@ export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunn
return true;
}
@Option({
flags: '--verbose',
description: 'Verbose output',
})
parseVerbose() {
return true;
}
@Option({
flags: '--company',
description: 'Enforce unique constraints on company domainName',
@ -76,15 +85,7 @@ export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunn
this.logger.log(`Running command for workspace ${workspaceId}`);
try {
await this.enforceUniqueConstraintsForWorkspace(
workspaceId,
options.dryRun ?? false,
options,
);
await this.twentyORMGlobalManager.destroyDataSourceForWorkspace(
workspaceId,
);
await this.enforceUniqueConstraintsForWorkspace(workspaceId, options);
} catch (error) {
this.logger.log(
chalk.red(
@ -104,30 +105,31 @@ export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunn
private async enforceUniqueConstraintsForWorkspace(
workspaceId: string,
dryRun: boolean,
options: EnforceUniqueConstraintsCommandOptions,
): Promise<void> {
await this.enforceUniquePersonEmail(workspaceId, dryRun);
if (options.person) {
await this.enforceUniquePersonEmail(workspaceId, options);
}
if (options.company) {
await this.enforceUniqueCompanyDomainName(workspaceId, dryRun);
await this.enforceUniqueCompanyDomainName(workspaceId, options);
}
if (options.viewField) {
await this.enforceUniqueViewField(workspaceId, dryRun);
await this.enforceUniqueViewField(workspaceId, options);
}
if (options.viewSort) {
await this.enforceUniqueViewSort(workspaceId, dryRun);
await this.enforceUniqueViewSort(workspaceId, options);
}
}
private async enforceUniqueCompanyDomainName(
workspaceId: string,
dryRun: boolean,
options: EnforceUniqueConstraintsCommandOptions,
): Promise<void> {
const companyRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
'company',
false,
);
const duplicates = await companyRepository
@ -156,28 +158,31 @@ export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunn
for (let i = 1; i < companies.length; i++) {
const newdomainNamePrimaryLinkUrl = `${company_domainNamePrimaryLinkUrl}${i}`;
if (!dryRun) {
if (!options.dryRun) {
await companyRepository.update(companies[i].id, {
domainNamePrimaryLinkUrl: newdomainNamePrimaryLinkUrl,
});
}
this.logger.log(
chalk.yellow(
`Updated company ${companies[i].id} domainName from ${company_domainNamePrimaryLinkUrl} to ${newdomainNamePrimaryLinkUrl}`,
),
);
if (options.verbose) {
this.logger.log(
chalk.yellow(
`Updated company ${companies[i].id} domainName from ${company_domainNamePrimaryLinkUrl} to ${newdomainNamePrimaryLinkUrl}`,
),
);
}
}
}
}
private async enforceUniquePersonEmail(
workspaceId: string,
dryRun: boolean,
options: EnforceUniqueConstraintsCommandOptions,
): Promise<void> {
const personRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
'person',
false,
);
const duplicates = await personRepository
@ -208,28 +213,31 @@ export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunn
? `${person_emailsPrimaryEmail.split('@')[0]}+${i}@${person_emailsPrimaryEmail.split('@')[1]}`
: `${person_emailsPrimaryEmail}+${i}`;
if (!dryRun) {
if (!options.dryRun) {
await personRepository.update(persons[i].id, {
emailsPrimaryEmail: newEmail,
});
}
this.logger.log(
chalk.yellow(
`Updated person ${persons[i].id} emailsPrimaryEmail from ${person_emailsPrimaryEmail} to ${newEmail}`,
),
);
if (options.verbose) {
this.logger.log(
chalk.yellow(
`Updated person ${persons[i].id} emailsPrimaryEmail from ${person_emailsPrimaryEmail} to ${newEmail}`,
),
);
}
}
}
}
private async enforceUniqueViewField(
workspaceId: string,
dryRun: boolean,
options: EnforceUniqueConstraintsCommandOptions,
): Promise<void> {
const viewFieldRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
'viewField',
false,
);
const duplicates = await viewFieldRepository
@ -249,26 +257,29 @@ export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunn
});
for (let i = 1; i < viewFields.length; i++) {
if (!dryRun) {
if (!options.dryRun) {
await viewFieldRepository.softDelete(viewFields[i].id);
}
this.logger.log(
chalk.yellow(
`Soft deleted duplicate ViewField ${viewFields[i].id} for fieldMetadataId ${fieldMetadataId} and viewId ${viewId}`,
),
);
if (options.verbose) {
this.logger.log(
chalk.yellow(
`Soft deleted duplicate ViewField ${viewFields[i].id} for fieldMetadataId ${fieldMetadataId} and viewId ${viewId}`,
),
);
}
}
}
}
private async enforceUniqueViewSort(
workspaceId: string,
dryRun: boolean,
options: EnforceUniqueConstraintsCommandOptions,
): Promise<void> {
const viewSortRepository =
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
workspaceId,
'viewSort',
false,
);
const duplicates = await viewSortRepository
@ -288,14 +299,16 @@ export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunn
});
for (let i = 1; i < viewSorts.length; i++) {
if (!dryRun) {
if (!options.dryRun) {
await viewSortRepository.softDelete(viewSorts[i].id);
}
this.logger.log(
chalk.yellow(
`Soft deleted duplicate ViewSort ${viewSorts[i].id} for fieldMetadataId ${fieldMetadataId} and viewId ${viewId}`,
),
);
if (options.verbose) {
this.logger.log(
chalk.yellow(
`Soft deleted duplicate ViewSort ${viewSorts[i].id} for fieldMetadataId ${fieldMetadataId} and viewId ${viewId}`,
),
);
}
}
}
}

View File

@ -4,8 +4,10 @@ import { Command } from 'nest-commander';
import { Repository } from 'typeorm';
import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
import { UpdateRichTextSearchVectorCommand } from 'src/database/commands/upgrade-version/0-32/0-33/0-33-update-rich-text-search-vector-expression';
import { EnforceUniqueConstraintsCommand } from 'src/database/commands/upgrade-version/0-33/0-33-enforce-unique-constraints.command';
import { UpdateRichTextSearchVectorCommand } from 'src/database/commands/upgrade-version/0-33/0-33-update-rich-text-search-vector-expression';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
interface UpdateTo0_33CommandOptions {
workspaceId?: string;
@ -20,6 +22,8 @@ export class UpgradeTo0_33Command extends ActiveWorkspacesCommandRunner {
@InjectRepository(Workspace, 'core')
protected readonly workspaceRepository: Repository<Workspace>,
private readonly updateRichTextSearchVectorCommand: UpdateRichTextSearchVectorCommand,
private readonly enforceUniqueConstraintsCommand: EnforceUniqueConstraintsCommand,
private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
) {
super(workspaceRepository);
}
@ -29,6 +33,25 @@ export class UpgradeTo0_33Command extends ActiveWorkspacesCommandRunner {
options: UpdateTo0_33CommandOptions,
workspaceIds: string[],
): Promise<void> {
await this.enforceUniqueConstraintsCommand.executeActiveWorkspacesCommand(
passedParam,
{
...options,
company: true,
person: true,
viewField: true,
viewSort: true,
},
workspaceIds,
);
await this.syncWorkspaceMetadataCommand.executeActiveWorkspacesCommand(
passedParam,
{
...options,
force: true,
},
workspaceIds,
);
await this.updateRichTextSearchVectorCommand.executeActiveWorkspacesCommand(
passedParam,
options,

View File

@ -1,8 +1,9 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { UpdateRichTextSearchVectorCommand } from 'src/database/commands/upgrade-version/0-32/0-33/0-33-update-rich-text-search-vector-expression';
import { UpgradeTo0_33Command } from 'src/database/commands/upgrade-version/0-32/0-33/0-33-upgrade-version.command';
import { EnforceUniqueConstraintsCommand } from 'src/database/commands/upgrade-version/0-33/0-33-enforce-unique-constraints.command';
import { UpdateRichTextSearchVectorCommand } from 'src/database/commands/upgrade-version/0-33/0-33-update-rich-text-search-vector-expression';
import { UpgradeTo0_33Command } from 'src/database/commands/upgrade-version/0-33/0-33-upgrade-version.command';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
@ -21,6 +22,10 @@ import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manage
SearchModule,
WorkspaceMigrationRunnerModule,
],
providers: [UpgradeTo0_33Command, UpdateRichTextSearchVectorCommand],
providers: [
UpgradeTo0_33Command,
UpdateRichTextSearchVectorCommand,
EnforceUniqueConstraintsCommand,
],
})
export class UpgradeTo0_33CommandModule {}

View File

@ -25,7 +25,9 @@ export function WorkspaceIndex(
);
metadataArgsStorage.addIndexes({
name: `IDX_${generateDeterministicIndexName([
name: `IDX_${
options?.isUnique ? 'UNIQUE_' : ''
}${generateDeterministicIndexName([
convertClassNameToObjectMetadataName(target.name),
...columns,
])}`,

View File

@ -201,22 +201,31 @@ export class WorkspaceMigrationRunnerService {
for (const index of indexes) {
switch (index.action) {
case WorkspaceMigrationIndexActionType.CREATE:
if (isDefined(index.type) && index.type !== IndexType.BTREE) {
const quotedColumns = index.columns.map((column) => `"${column}"`);
try {
if (isDefined(index.type) && index.type !== IndexType.BTREE) {
const quotedColumns = index.columns.map(
(column) => `"${column}"`,
);
await queryRunner.query(`
await queryRunner.query(`
CREATE INDEX "${index.name}" ON "${schemaName}"."${tableName}" USING ${index.type} (${quotedColumns.join(', ')})
`);
} else {
await queryRunner.createIndex(
`${schemaName}.${tableName}`,
new TableIndex({
name: index.name,
columnNames: index.columns,
isUnique: index.isUnique,
where: index.where ?? undefined,
}),
);
} else {
await queryRunner.createIndex(
`${schemaName}.${tableName}`,
new TableIndex({
name: index.name,
columnNames: index.columns,
isUnique: index.isUnique,
where: index.where ?? undefined,
}),
);
}
} catch (error) {
// Ignore error if index already exists
if (error.code === '42P07') {
continue;
}
}
break;
case WorkspaceMigrationIndexActionType.DROP:
@ -461,7 +470,10 @@ export class WorkspaceMigrationRunnerService {
),
isArray: migrationColumn.currentColumnDefinition.isArray,
isNullable: migrationColumn.currentColumnDefinition.isNullable,
isUnique: migrationColumn.currentColumnDefinition.isUnique,
/* For now unique constraints are created at a higher level
as we need to handle soft-delete and a bug on empty strings
*/
// isUnique: migrationColumn.currentColumnDefinition.isUnique,
}),
new TableColumn({
name: migrationColumn.alteredColumnDefinition.columnName,
@ -474,7 +486,10 @@ export class WorkspaceMigrationRunnerService {
isNullable: migrationColumn.alteredColumnDefinition.isNullable,
asExpression: migrationColumn.alteredColumnDefinition.asExpression,
generatedType: migrationColumn.alteredColumnDefinition.generatedType,
isUnique: migrationColumn.alteredColumnDefinition.isUnique,
/* For now unique constraints are created at a higher level
as we need to handle soft-delete and a bug on empty strings
*/
// isUnique: migrationColumn.alteredColumnDefinition.isUnique,
}),
);
}

View File

@ -21,6 +21,7 @@ import { WorkspaceField } from 'src/engine/twenty-orm/decorators/workspace-field
import { WorkspaceIsDeprecated } from 'src/engine/twenty-orm/decorators/workspace-is-deprecated.decorator';
import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace-is-nullable.decorator';
import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator';
import { WorkspaceIsUnique } from 'src/engine/twenty-orm/decorators/workspace-is-unique.decorator';
import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace-join-column.decorator';
import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator';
import { COMPANY_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
@ -75,10 +76,7 @@ export class CompanyWorkspaceEntity extends BaseWorkspaceEntity {
'The company website URL. We use this url to fetch the company icon',
icon: 'IconLink',
})
/*
TODO: add soon once we've confirmed it's stabled
@WorkspaceIsUnique()
*/
[DOMAIN_NAME_FIELD_NAME]?: LinksMetadata;
@WorkspaceField({

View File

@ -22,6 +22,7 @@ import { WorkspaceField } from 'src/engine/twenty-orm/decorators/workspace-field
import { WorkspaceIsDeprecated } from 'src/engine/twenty-orm/decorators/workspace-is-deprecated.decorator';
import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace-is-nullable.decorator';
import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator';
import { WorkspaceIsUnique } from 'src/engine/twenty-orm/decorators/workspace-is-unique.decorator';
import { WorkspaceJoinColumn } from 'src/engine/twenty-orm/decorators/workspace-join-column.decorator';
import { WorkspaceRelation } from 'src/engine/twenty-orm/decorators/workspace-relation.decorator';
import { PERSON_STANDARD_FIELD_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
@ -80,7 +81,7 @@ export class PersonWorkspaceEntity extends BaseWorkspaceEntity {
description: 'Contacts Emails',
icon: 'IconMail',
})
// @WorkspaceIsUnique()
@WorkspaceIsUnique()
[EMAILS_FIELD_NAME]: EmailsMetadata;
@WorkspaceField({

View File

@ -3,6 +3,7 @@ import { RelationMetadataType } from 'src/engine/metadata-modules/relation-metad
import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity';
import { WorkspaceEntity } from 'src/engine/twenty-orm/decorators/workspace-entity.decorator';
import { WorkspaceField } from 'src/engine/twenty-orm/decorators/workspace-field.decorator';
import { WorkspaceIndex } from 'src/engine/twenty-orm/decorators/workspace-index.decorator';
import { WorkspaceIsNotAuditLogged } from 'src/engine/twenty-orm/decorators/workspace-is-not-audit-logged.decorator';
import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace-is-nullable.decorator';
import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator';
@ -23,12 +24,10 @@ import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.work
})
@WorkspaceIsNotAuditLogged()
@WorkspaceIsSystem()
/*
TODO: add soon once we've confirmed it's stabled
@WorkspaceIndex(['fieldMetadataId', 'viewId'], {
isUnique: true,
indexWhereClause: '"deletedAt" IS NULL',
})*/
})
export class ViewFieldWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceField({
standardId: VIEW_FIELD_STANDARD_FIELD_IDS.fieldMetadataId,

View File

@ -5,6 +5,7 @@ import { RelationMetadataType } from 'src/engine/metadata-modules/relation-metad
import { BaseWorkspaceEntity } from 'src/engine/twenty-orm/base.workspace-entity';
import { WorkspaceEntity } from 'src/engine/twenty-orm/decorators/workspace-entity.decorator';
import { WorkspaceField } from 'src/engine/twenty-orm/decorators/workspace-field.decorator';
import { WorkspaceIndex } from 'src/engine/twenty-orm/decorators/workspace-index.decorator';
import { WorkspaceIsNotAuditLogged } from 'src/engine/twenty-orm/decorators/workspace-is-not-audit-logged.decorator';
import { WorkspaceIsNullable } from 'src/engine/twenty-orm/decorators/workspace-is-nullable.decorator';
import { WorkspaceIsSystem } from 'src/engine/twenty-orm/decorators/workspace-is-system.decorator';
@ -25,12 +26,10 @@ import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.work
})
@WorkspaceIsNotAuditLogged()
@WorkspaceIsSystem()
/*
TODO: add soon once we've confirmed it's stabled
@WorkspaceIndex(['fieldMetadataId', 'viewId'], {
isUnique: true,
indexWhereClause: '"deletedAt" IS NULL',
})*/
})
export class ViewSortWorkspaceEntity extends BaseWorkspaceEntity {
@WorkspaceField({
standardId: VIEW_SORT_STANDARD_FIELD_IDS.fieldMetadataId,