Enforce unique constraints for 0.33 (#8645)
## Context - Fixing folder structure where 0.33 was inside 0.32. - Updating the command so it runs enforce uniqueness on all tables - Updating workspaces entities so the sync metadata adds the index
This commit is contained in:
Weiko
@ -0,0 +1,315 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import chalk from 'chalk';
|
||||
import { Command, Option } from 'nest-commander';
|
||||
import { IsNull, Repository } from 'typeorm';
|
||||
|
||||
import {
|
||||
ActiveWorkspacesCommandOptions,
|
||||
ActiveWorkspacesCommandRunner,
|
||||
} from 'src/database/commands/active-workspaces.command';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
|
||||
interface EnforceUniqueConstraintsCommandOptions
|
||||
extends ActiveWorkspacesCommandOptions {
|
||||
person?: boolean;
|
||||
company?: boolean;
|
||||
viewField?: boolean;
|
||||
viewSort?: boolean;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
@Command({
|
||||
name: 'upgrade-0.33:enforce-unique-constraints',
|
||||
description:
|
||||
'Enforce unique constraints on company domainName, person emailsPrimaryEmail, ViewField, and ViewSort',
|
||||
})
|
||||
export class EnforceUniqueConstraintsCommand extends ActiveWorkspacesCommandRunner {
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
protected readonly workspaceRepository: Repository<Workspace>,
|
||||
private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
|
||||
) {
|
||||
super(workspaceRepository);
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--person',
|
||||
description: 'Enforce unique constraints on person emailsPrimaryEmail',
|
||||
})
|
||||
parsePerson() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--verbose',
|
||||
description: 'Verbose output',
|
||||
})
|
||||
parseVerbose() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--company',
|
||||
description: 'Enforce unique constraints on company domainName',
|
||||
})
|
||||
parseCompany() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--view-field',
|
||||
description: 'Enforce unique constraints on ViewField',
|
||||
})
|
||||
parseViewField() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Option({
|
||||
flags: '--view-sort',
|
||||
description: 'Enforce unique constraints on ViewSort',
|
||||
})
|
||||
parseViewSort() {
|
||||
return true;
|
||||
}
|
||||
|
||||
async executeActiveWorkspacesCommand(
|
||||
_passedParam: string[],
|
||||
options: EnforceUniqueConstraintsCommandOptions,
|
||||
workspaceIds: string[],
|
||||
): Promise<void> {
|
||||
this.logger.log('Running command to enforce unique constraints');
|
||||
|
||||
for (const workspaceId of workspaceIds) {
|
||||
this.logger.log(`Running command for workspace ${workspaceId}`);
|
||||
|
||||
try {
|
||||
await this.enforceUniqueConstraintsForWorkspace(workspaceId, options);
|
||||
} catch (error) {
|
||||
this.logger.log(
|
||||
chalk.red(
|
||||
`Running command on workspace ${workspaceId} failed with error: ${error}, ${error.stack}`,
|
||||
),
|
||||
);
|
||||
continue;
|
||||
} finally {
|
||||
this.logger.log(
|
||||
chalk.green(`Finished running command for workspace ${workspaceId}.`),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(chalk.green(`Command completed!`));
|
||||
}
|
||||
|
||||
private async enforceUniqueConstraintsForWorkspace(
|
||||
workspaceId: string,
|
||||
options: EnforceUniqueConstraintsCommandOptions,
|
||||
): Promise<void> {
|
||||
if (options.person) {
|
||||
await this.enforceUniquePersonEmail(workspaceId, options);
|
||||
}
|
||||
if (options.company) {
|
||||
await this.enforceUniqueCompanyDomainName(workspaceId, options);
|
||||
}
|
||||
if (options.viewField) {
|
||||
await this.enforceUniqueViewField(workspaceId, options);
|
||||
}
|
||||
if (options.viewSort) {
|
||||
await this.enforceUniqueViewSort(workspaceId, options);
|
||||
}
|
||||
}
|
||||
|
||||
private async enforceUniqueCompanyDomainName(
|
||||
workspaceId: string,
|
||||
options: EnforceUniqueConstraintsCommandOptions,
|
||||
): Promise<void> {
|
||||
const companyRepository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
|
||||
workspaceId,
|
||||
'company',
|
||||
false,
|
||||
);
|
||||
|
||||
const duplicates = await companyRepository
|
||||
.createQueryBuilder('company')
|
||||
.select('company.domainNamePrimaryLinkUrl')
|
||||
.addSelect('COUNT(*)', 'count')
|
||||
.where('company.deletedAt IS NULL')
|
||||
.where('company.domainNamePrimaryLinkUrl IS NOT NULL')
|
||||
.where("company.domainNamePrimaryLinkUrl != ''")
|
||||
.groupBy('company.domainNamePrimaryLinkUrl')
|
||||
.having('COUNT(*) > 1')
|
||||
.getRawMany();
|
||||
|
||||
for (const duplicate of duplicates) {
|
||||
const { company_domainNamePrimaryLinkUrl } = duplicate;
|
||||
const companies = await companyRepository.find({
|
||||
where: {
|
||||
domainName: {
|
||||
primaryLinkUrl: company_domainNamePrimaryLinkUrl,
|
||||
},
|
||||
deletedAt: IsNull(),
|
||||
},
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
|
||||
for (let i = 1; i < companies.length; i++) {
|
||||
const newdomainNamePrimaryLinkUrl = `${company_domainNamePrimaryLinkUrl}${i}`;
|
||||
|
||||
if (!options.dryRun) {
|
||||
await companyRepository.update(companies[i].id, {
|
||||
domainNamePrimaryLinkUrl: newdomainNamePrimaryLinkUrl,
|
||||
});
|
||||
}
|
||||
if (options.verbose) {
|
||||
this.logger.log(
|
||||
chalk.yellow(
|
||||
`Updated company ${companies[i].id} domainName from ${company_domainNamePrimaryLinkUrl} to ${newdomainNamePrimaryLinkUrl}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async enforceUniquePersonEmail(
|
||||
workspaceId: string,
|
||||
options: EnforceUniqueConstraintsCommandOptions,
|
||||
): Promise<void> {
|
||||
const personRepository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
|
||||
workspaceId,
|
||||
'person',
|
||||
false,
|
||||
);
|
||||
|
||||
const duplicates = await personRepository
|
||||
.createQueryBuilder('person')
|
||||
.select('person.emailsPrimaryEmail')
|
||||
.addSelect('COUNT(*)', 'count')
|
||||
.where('person.deletedAt IS NULL')
|
||||
.where('person.emailsPrimaryEmail IS NOT NULL')
|
||||
.where("person.emailsPrimaryEmail != ''")
|
||||
.groupBy('person.emailsPrimaryEmail')
|
||||
.having('COUNT(*) > 1')
|
||||
.getRawMany();
|
||||
|
||||
for (const duplicate of duplicates) {
|
||||
const { person_emailsPrimaryEmail } = duplicate;
|
||||
const persons = await personRepository.find({
|
||||
where: {
|
||||
emails: {
|
||||
primaryEmail: person_emailsPrimaryEmail,
|
||||
},
|
||||
deletedAt: IsNull(),
|
||||
},
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
|
||||
for (let i = 1; i < persons.length; i++) {
|
||||
const newEmail = person_emailsPrimaryEmail?.includes('@')
|
||||
? `${person_emailsPrimaryEmail.split('@')[0]}+${i}@${person_emailsPrimaryEmail.split('@')[1]}`
|
||||
: `${person_emailsPrimaryEmail}+${i}`;
|
||||
|
||||
if (!options.dryRun) {
|
||||
await personRepository.update(persons[i].id, {
|
||||
emailsPrimaryEmail: newEmail,
|
||||
});
|
||||
}
|
||||
if (options.verbose) {
|
||||
this.logger.log(
|
||||
chalk.yellow(
|
||||
`Updated person ${persons[i].id} emailsPrimaryEmail from ${person_emailsPrimaryEmail} to ${newEmail}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async enforceUniqueViewField(
|
||||
workspaceId: string,
|
||||
options: EnforceUniqueConstraintsCommandOptions,
|
||||
): Promise<void> {
|
||||
const viewFieldRepository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
|
||||
workspaceId,
|
||||
'viewField',
|
||||
false,
|
||||
);
|
||||
|
||||
const duplicates = await viewFieldRepository
|
||||
.createQueryBuilder('viewField')
|
||||
.select(['viewField.fieldMetadataId', 'viewField.viewId'])
|
||||
.addSelect('COUNT(*)', 'count')
|
||||
.where('viewField.deletedAt IS NULL')
|
||||
.groupBy('viewField.fieldMetadataId, viewField.viewId')
|
||||
.having('COUNT(*) > 1')
|
||||
.getRawMany();
|
||||
|
||||
for (const duplicate of duplicates) {
|
||||
const { fieldMetadataId, viewId } = duplicate;
|
||||
const viewFields = await viewFieldRepository.find({
|
||||
where: { fieldMetadataId, viewId, deletedAt: IsNull() },
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
|
||||
for (let i = 1; i < viewFields.length; i++) {
|
||||
if (!options.dryRun) {
|
||||
await viewFieldRepository.softDelete(viewFields[i].id);
|
||||
}
|
||||
if (options.verbose) {
|
||||
this.logger.log(
|
||||
chalk.yellow(
|
||||
`Soft deleted duplicate ViewField ${viewFields[i].id} for fieldMetadataId ${fieldMetadataId} and viewId ${viewId}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async enforceUniqueViewSort(
|
||||
workspaceId: string,
|
||||
options: EnforceUniqueConstraintsCommandOptions,
|
||||
): Promise<void> {
|
||||
const viewSortRepository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace(
|
||||
workspaceId,
|
||||
'viewSort',
|
||||
false,
|
||||
);
|
||||
|
||||
const duplicates = await viewSortRepository
|
||||
.createQueryBuilder('viewSort')
|
||||
.select(['viewSort.fieldMetadataId', 'viewSort.viewId'])
|
||||
.addSelect('COUNT(*)', 'count')
|
||||
.where('viewSort.deletedAt IS NULL')
|
||||
.groupBy('viewSort.fieldMetadataId, viewSort.viewId')
|
||||
.having('COUNT(*) > 1')
|
||||
.getRawMany();
|
||||
|
||||
for (const duplicate of duplicates) {
|
||||
const { fieldMetadataId, viewId } = duplicate;
|
||||
const viewSorts = await viewSortRepository.find({
|
||||
where: { fieldMetadataId, viewId, deletedAt: IsNull() },
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
|
||||
for (let i = 1; i < viewSorts.length; i++) {
|
||||
if (!options.dryRun) {
|
||||
await viewSortRepository.softDelete(viewSorts[i].id);
|
||||
}
|
||||
if (options.verbose) {
|
||||
this.logger.log(
|
||||
chalk.yellow(
|
||||
`Soft deleted duplicate ViewSort ${viewSorts[i].id} for fieldMetadataId ${fieldMetadataId} and viewId ${viewId}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,109 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import chalk from 'chalk';
|
||||
import { Command } from 'nest-commander';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import {
|
||||
ActiveWorkspacesCommandOptions,
|
||||
ActiveWorkspacesCommandRunner,
|
||||
} from 'src/database/commands/active-workspaces.command';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import {
|
||||
FieldMetadataEntity,
|
||||
FieldMetadataType,
|
||||
} from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { SearchService } from 'src/engine/metadata-modules/search/search.service';
|
||||
import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service';
|
||||
import {
|
||||
NOTE_STANDARD_FIELD_IDS,
|
||||
TASK_STANDARD_FIELD_IDS,
|
||||
} from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
|
||||
import { FieldTypeAndNameMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util';
|
||||
import { SEARCH_FIELDS_FOR_NOTES } from 'src/modules/note/standard-objects/note.workspace-entity';
|
||||
import { SEARCH_FIELDS_FOR_TASK } from 'src/modules/task/standard-objects/task.workspace-entity';
|
||||
|
||||
@Command({
|
||||
name: 'fix-0.33:update-rich-text-expression',
|
||||
description: 'Update rich text (note and task) search vector expressions',
|
||||
})
|
||||
export class UpdateRichTextSearchVectorCommand extends ActiveWorkspacesCommandRunner {
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
protected readonly workspaceRepository: Repository<Workspace>,
|
||||
@InjectRepository(FieldMetadataEntity, 'metadata')
|
||||
private readonly fieldMetadataRepository: Repository<FieldMetadataEntity>,
|
||||
private readonly searchService: SearchService,
|
||||
private readonly workspaceMigrationRunnerService: WorkspaceMigrationRunnerService,
|
||||
) {
|
||||
super(workspaceRepository);
|
||||
}
|
||||
|
||||
async executeActiveWorkspacesCommand(
|
||||
_passedParam: string[],
|
||||
_options: ActiveWorkspacesCommandOptions,
|
||||
workspaceIds: string[],
|
||||
): Promise<void> {
|
||||
this.logger.log('Running command to fix migration');
|
||||
|
||||
for (const workspaceId of workspaceIds) {
|
||||
this.logger.log(`Running command for workspace ${workspaceId}`);
|
||||
|
||||
try {
|
||||
const searchVectorFields = await this.fieldMetadataRepository.findBy({
|
||||
workspaceId: workspaceId,
|
||||
type: FieldMetadataType.TS_VECTOR,
|
||||
});
|
||||
|
||||
for (const searchVectorField of searchVectorFields) {
|
||||
let fieldsUsedForSearch: FieldTypeAndNameMetadata[] = [];
|
||||
let objectNameForLog = '';
|
||||
|
||||
switch (searchVectorField.standardId) {
|
||||
case NOTE_STANDARD_FIELD_IDS.searchVector: {
|
||||
fieldsUsedForSearch = SEARCH_FIELDS_FOR_NOTES;
|
||||
objectNameForLog = 'Note';
|
||||
break;
|
||||
}
|
||||
case TASK_STANDARD_FIELD_IDS.searchVector: {
|
||||
fieldsUsedForSearch = SEARCH_FIELDS_FOR_TASK;
|
||||
objectNameForLog = 'Task';
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (fieldsUsedForSearch.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Updating searchVector for ${searchVectorField.objectMetadataId} (${objectNameForLog})...`,
|
||||
);
|
||||
|
||||
await this.searchService.updateSearchVector(
|
||||
searchVectorField.objectMetadataId,
|
||||
fieldsUsedForSearch,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
await this.workspaceMigrationRunnerService.executeMigrationFromPendingMigrations(
|
||||
workspaceId,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.log(
|
||||
chalk.red(
|
||||
`Running command on workspace ${workspaceId} failed with error: ${error}`,
|
||||
),
|
||||
);
|
||||
continue;
|
||||
} finally {
|
||||
this.logger.log(
|
||||
chalk.green(`Finished running command for workspace ${workspaceId}.`),
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(chalk.green(`Command completed!`));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,61 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Command } from 'nest-commander';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { ActiveWorkspacesCommandRunner } from 'src/database/commands/active-workspaces.command';
|
||||
import { EnforceUniqueConstraintsCommand } from 'src/database/commands/upgrade-version/0-33/0-33-enforce-unique-constraints.command';
|
||||
import { UpdateRichTextSearchVectorCommand } from 'src/database/commands/upgrade-version/0-33/0-33-update-rich-text-search-vector-expression';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { SyncWorkspaceMetadataCommand } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/sync-workspace-metadata.command';
|
||||
|
||||
interface UpdateTo0_33CommandOptions {
|
||||
workspaceId?: string;
|
||||
}
|
||||
|
||||
@Command({
|
||||
name: 'upgrade-0.33',
|
||||
description: 'Upgrade to 0.33',
|
||||
})
|
||||
export class UpgradeTo0_33Command extends ActiveWorkspacesCommandRunner {
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
protected readonly workspaceRepository: Repository<Workspace>,
|
||||
private readonly updateRichTextSearchVectorCommand: UpdateRichTextSearchVectorCommand,
|
||||
private readonly enforceUniqueConstraintsCommand: EnforceUniqueConstraintsCommand,
|
||||
private readonly syncWorkspaceMetadataCommand: SyncWorkspaceMetadataCommand,
|
||||
) {
|
||||
super(workspaceRepository);
|
||||
}
|
||||
|
||||
async executeActiveWorkspacesCommand(
|
||||
passedParam: string[],
|
||||
options: UpdateTo0_33CommandOptions,
|
||||
workspaceIds: string[],
|
||||
): Promise<void> {
|
||||
await this.enforceUniqueConstraintsCommand.executeActiveWorkspacesCommand(
|
||||
passedParam,
|
||||
{
|
||||
...options,
|
||||
company: true,
|
||||
person: true,
|
||||
viewField: true,
|
||||
viewSort: true,
|
||||
},
|
||||
workspaceIds,
|
||||
);
|
||||
await this.syncWorkspaceMetadataCommand.executeActiveWorkspacesCommand(
|
||||
passedParam,
|
||||
{
|
||||
...options,
|
||||
force: true,
|
||||
},
|
||||
workspaceIds,
|
||||
);
|
||||
await this.updateRichTextSearchVectorCommand.executeActiveWorkspacesCommand(
|
||||
passedParam,
|
||||
options,
|
||||
workspaceIds,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { EnforceUniqueConstraintsCommand } from 'src/database/commands/upgrade-version/0-33/0-33-enforce-unique-constraints.command';
|
||||
import { UpdateRichTextSearchVectorCommand } from 'src/database/commands/upgrade-version/0-33/0-33-update-rich-text-search-vector-expression';
|
||||
import { UpgradeTo0_33Command } from 'src/database/commands/upgrade-version/0-33/0-33-upgrade-version.command';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
|
||||
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
|
||||
import { SearchModule } from 'src/engine/metadata-modules/search/search.module';
|
||||
import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module';
|
||||
import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([Workspace], 'core'),
|
||||
TypeOrmModule.forFeature(
|
||||
[ObjectMetadataEntity, FieldMetadataEntity],
|
||||
'metadata',
|
||||
),
|
||||
WorkspaceSyncMetadataCommandsModule,
|
||||
SearchModule,
|
||||
WorkspaceMigrationRunnerModule,
|
||||
],
|
||||
providers: [
|
||||
UpgradeTo0_33Command,
|
||||
UpdateRichTextSearchVectorCommand,
|
||||
EnforceUniqueConstraintsCommand,
|
||||
],
|
||||
})
|
||||
export class UpgradeTo0_33CommandModule {}
|
||||
Reference in New Issue
Block a user