diff --git a/server/src/database/commands/clean-inactive-workspaces.command.ts b/server/src/database/commands/clean-inactive-workspaces.command.ts index e013d0e6f..4c2ed752a 100644 --- a/server/src/database/commands/clean-inactive-workspaces.command.ts +++ b/server/src/database/commands/clean-inactive-workspaces.command.ts @@ -25,17 +25,19 @@ interface ActivityReport { displayName: string; maxUpdatedAt: string; inactiveDays: number; -} - -interface SameAsSeedWorkspace { - displayName: string; + sameAsSeed: boolean; } interface DataCleanResults { - activityReport: { [key: string]: ActivityReport }; - sameAsSeedWorkspaces: { [key: string]: SameAsSeedWorkspace }; + [key: string]: ActivityReport; } +const formattedPipelineStagesSeed = pipelineStagesSeed.map((pipelineStage) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { position, ...rest } = pipelineStage; + return rest; +}); + @Command({ name: 'workspaces:clean-inactive', description: 'Clean inactive workspaces from the public database schema', @@ -83,7 +85,7 @@ export class DataCleanInactiveCommand extends CommandRunner { return Boolean(val); } - // We look for public tables which contains workspaceId and updatedAt columns + // We look for public tables which contain workspaceId and updatedAt columns and exist in production database getRelevantTables() { return Object.keys(this.prismaService.client).filter( (name) => @@ -96,36 +98,58 @@ export class DataCleanInactiveCommand extends CommandRunner { ); } - async getTableMaxUpdatedAt(table, workspace) { - return await this.prismaService.client[table].aggregate({ - _max: { updatedAt: true }, - where: { workspaceId: { equals: workspace.id } }, - }); + async getMaxUpdatedAtForAllWorkspaces(tables, workspaces) { + const result = {}; + for (const table of tables) { + result[table] = {}; + const groupByWorkspaces = await this.prismaService.client[table].groupBy({ + by: ['workspaceId'], + _max: { updatedAt: true }, + where: { + workspaceId: { in: workspaces.map((workspace) => workspace.id) }, + }, + }); + for (const groupByWorkspace of groupByWorkspaces) { + result[table][groupByWorkspace.workspaceId] = + groupByWorkspace._max.updatedAt; + } + } + return result; } - async addMaxUpdatedAtToWorkspaces(result, workspace, table) { - const newUpdatedAt = await this.getTableMaxUpdatedAt(table, workspace); - if (!result.activityReport[workspace.id]) { - result.activityReport[workspace.id] = { + async addMaxUpdatedAtToWorkspaces( + result, + workspace, + table, + maxUpdatedAtForAllWorkspaces, + ) { + const newUpdatedAt = maxUpdatedAtForAllWorkspaces[table][workspace.id]; + if (!result[workspace.id]) { + result[workspace.id] = { displayName: workspace.displayName, maxUpdatedAt: null, }; } if ( newUpdatedAt && - newUpdatedAt._max.updatedAt && - new Date(result.activityReport[workspace.id].maxUpdatedAt) < - new Date(newUpdatedAt._max.updatedAt) + new Date(result[workspace.id].maxUpdatedAt) < new Date(newUpdatedAt) ) { - result.activityReport[workspace.id].maxUpdatedAt = - newUpdatedAt._max.updatedAt; + result[workspace.id].maxUpdatedAt = newUpdatedAt; } } - - async detectWorkspacesWithSeedDataOnly(result, workspace) { + async getSeedTableData(workspaces) { + const where = { + workspaceId: { in: workspaces.map((workspace) => workspace.id) }, + }; const companies = await this.prismaService.client.company.findMany({ - select: { name: true, domainName: true, address: true, employees: true }, - where: { workspaceId: { equals: workspace.id } }, + select: { + name: true, + domainName: true, + address: true, + employees: true, + workspaceId: true, + }, + where, }); const people = await this.prismaService.client.person.findMany({ select: { @@ -134,121 +158,178 @@ export class DataCleanInactiveCommand extends CommandRunner { city: true, email: true, avatarUrl: true, + workspaceId: true, }, - where: { workspaceId: { equals: workspace.id } }, + where, }); const pipelineStages = await this.prismaService.client.pipelineStage.findMany({ select: { name: true, color: true, - position: true, type: true, + workspaceId: true, }, - where: { workspaceId: { equals: workspace.id } }, + where, }); const pipelines = await this.prismaService.client.pipeline.findMany({ select: { name: true, icon: true, pipelineProgressableType: true, + workspaceId: true, }, - where: { workspaceId: { equals: workspace.id } }, + where, }); + return { + companies, + people, + pipelineStages, + pipelines, + }; + } + + async detectWorkspacesWithSeedDataOnly(result, workspace, seedTableData) { + const companies = seedTableData.companies.reduce((filtered, company) => { + if (company.workspaceId === workspace.id) { + delete company.workspaceId; + filtered.push(company); + } + return filtered; + }, []); + const people = seedTableData.people.reduce((filtered, person) => { + if (person.workspaceId === workspace.id) { + delete person.workspaceId; + filtered.push(person); + } + return filtered; + }, []); + const pipelineStages = seedTableData.pipelineStages.reduce( + (filtered, pipelineStage) => { + if (pipelineStage.workspaceId === workspace.id) { + delete pipelineStage.workspaceId; + filtered.push(pipelineStage); + } + return filtered; + }, + [], + ); + const pipelines = seedTableData.pipelines.reduce((filtered, pipeline) => { + if (pipeline.workspaceId === workspace.id) { + delete pipeline.workspaceId; + filtered.push(pipeline); + } + return filtered; + }, []); if ( isEqual(people, peopleSeed) && isEqual(companies, companiesSeed) && - isEqual(pipelineStages, pipelineStagesSeed) && + isEqual(pipelineStages, formattedPipelineStagesSeed) && isEqual(pipelines, [pipelinesSeed]) ) { - result.sameAsSeedWorkspaces[workspace.id] = { - displayName: workspace.displayName, - }; + result[workspace.id].sameAsSeed = true; + } else { + { + result[workspace.id].sameAsSeed = false; + } } } - async findInactiveWorkspaces(result, options) { + async getWorkspaces(options) { const where = options.workspaceId ? { id: { equals: options.workspaceId } } : {}; - const workspaces = await this.prismaService.client.workspace.findMany({ + return await this.prismaService.client.workspace.findMany({ where, + orderBy: [{ createdAt: 'asc' }], }); + } + + async findInactiveWorkspaces(workspaces, result) { const tables = this.getRelevantTables(); + const maxUpdatedAtForAllWorkspaces = + await this.getMaxUpdatedAtForAllWorkspaces(tables, workspaces); + const seedTableData = await this.getSeedTableData(workspaces); for (const workspace of workspaces) { - await this.detectWorkspacesWithSeedDataOnly(result, workspace); for (const table of tables) { - await this.addMaxUpdatedAtToWorkspaces(result, workspace, table); + await this.addMaxUpdatedAtToWorkspaces( + result, + workspace, + table, + maxUpdatedAtForAllWorkspaces, + ); } + await this.detectWorkspacesWithSeedDataOnly( + result, + workspace, + seedTableData, + ); } } filterResults(result, options) { - for (const workspaceId in result.activityReport) { + for (const workspaceId in result) { const timeDifferenceInSeconds = Math.abs( new Date().getTime() - - new Date(result.activityReport[workspaceId].maxUpdatedAt).getTime(), + new Date(result[workspaceId].maxUpdatedAt).getTime(), ); const timeDifferenceInDays = Math.ceil( timeDifferenceInSeconds / (1000 * 3600 * 24), ); - if (timeDifferenceInDays < options.sameAsSeedDays) { - delete result.sameAsSeedWorkspaces[workspaceId]; - } - if (timeDifferenceInDays < options.days) { - delete result.activityReport[workspaceId]; + if ( + timeDifferenceInDays < options.days && + (!result[workspaceId].sameAsSeed || + timeDifferenceInDays < options.sameAsSeedDays) + ) { + delete result[workspaceId]; } else { - result.activityReport[workspaceId].inactiveDays = timeDifferenceInDays; + result[workspaceId].inactiveDays = timeDifferenceInDays; } } } - async delete(result) { - if (Object.keys(result.activityReport).length) { - console.log('Deleting inactive workspaces'); + async delete(result, options) { + const workspaceCount = Object.keys(result).length; + if (workspaceCount) { + console.log( + `Deleting \x1b[36m${workspaceCount}\x1b[0m inactive since \x1b[36m${options.days} days\x1b[0m or same as seed since \x1b[36m${options.sameAsSeedDays} days\x1b[0m workspaces`, + ); } - for (const workspaceId in result.activityReport) { + let count = 1; + for (const workspaceId in result) { process.stdout.write(`- deleting ${workspaceId} ...`); await this.workspaceService.deleteWorkspace({ workspaceId, }); - console.log(' done!'); - } - if (Object.keys(result.sameAsSeedWorkspaces).length) { - console.log('Deleting same as Seed workspaces'); - } - for (const workspaceId in result.sameAsSeedWorkspaces) { - process.stdout.write(`- deleting ${workspaceId} ...`); - await this.workspaceService.deleteWorkspace({ - workspaceId, - }); - console.log(' done!'); + console.log( + ` done! ....... ${Math.floor((100 * count) / workspaceCount)}%`, + ); + count += 1; } } - displayResults(result) { - const workspacesToDelete = new Set(); - for (const workspaceId in result.activityReport) { - workspacesToDelete.add(workspaceId); - } - for (const workspaceId in result.sameAsSeedWorkspaces) { - workspacesToDelete.add(workspaceId); - } - console.log(`${workspacesToDelete.size} workspace(s) will be deleted:`); + displayResults(result, totalWorkspacesCount) { console.log(result); + console.log( + `${ + Object.keys(result).length + } out of ${totalWorkspacesCount} workspace(s) checked (${Math.floor( + (100 * Object.keys(result).length) / totalWorkspacesCount, + )}%) will be deleted`, + ); } async run( _passedParam: string[], options: DataCleanInactiveOptions, ): Promise { - const result: DataCleanResults = { - activityReport: {}, - sameAsSeedWorkspaces: {}, - }; - await this.findInactiveWorkspaces(result, options); + const result: DataCleanResults = {}; + const workspaces = await this.getWorkspaces(options); + const totalWorkspacesCount = workspaces.length; + console.log(totalWorkspacesCount, 'workspace(s) to analyse'); + await this.findInactiveWorkspaces(workspaces, result); this.filterResults(result, options); - this.displayResults(result); + this.displayResults(result, totalWorkspacesCount); if (!options.dryRun) { options = await this.inquiererService.ask('confirm', options); if (!options.confirmation) { @@ -257,7 +338,7 @@ export class DataCleanInactiveCommand extends CommandRunner { } } if (!options.dryRun) { - await this.delete(result); + await this.delete(result, options); } } } diff --git a/server/src/metadata/data-source/data-source.service.ts b/server/src/metadata/data-source/data-source.service.ts index 1cee57c26..60d761766 100644 --- a/server/src/metadata/data-source/data-source.service.ts +++ b/server/src/metadata/data-source/data-source.service.ts @@ -127,7 +127,8 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { const schemaAlreadyExists = await queryRunner.hasSchema(schemaName); if (!schemaAlreadyExists) { - throw new Error(`Schema ${schemaName} does not exist`); + await queryRunner.release(); + return; } await queryRunner.dropSchema(schemaName, true, true); diff --git a/server/src/metadata/field-metadata/services/field-metadata.service.ts b/server/src/metadata/field-metadata/services/field-metadata.service.ts index 4167909ee..f771b21b7 100644 --- a/server/src/metadata/field-metadata/services/field-metadata.service.ts +++ b/server/src/metadata/field-metadata/services/field-metadata.service.ts @@ -103,4 +103,8 @@ export class FieldMetadataService extends TypeOrmQueryService { return createdFieldMetadata; } + + public async deleteFieldsMetadata(workspaceId: string) { + await this.fieldMetadataRepository.delete({ workspaceId }); + } } diff --git a/server/src/metadata/metadata.datasource.ts b/server/src/metadata/metadata.datasource.ts index ab42b0cd6..4671d174a 100644 --- a/server/src/metadata/metadata.datasource.ts +++ b/server/src/metadata/metadata.datasource.ts @@ -15,7 +15,7 @@ export const typeORMMetadataModuleOptions: TypeOrmModuleOptions = { schema: 'metadata', entities: [__dirname + '/**/*.entity{.ts,.js}'], synchronize: false, - migrationsRun: true, + migrationsRun: false, migrationsTableName: '_typeorm_migrations', migrations: [__dirname + '/migrations/*{.ts,.js}'], }; diff --git a/server/src/metadata/object-metadata/services/object-metadata.service.ts b/server/src/metadata/object-metadata/services/object-metadata.service.ts index e08e920a1..3324d3b9c 100644 --- a/server/src/metadata/object-metadata/services/object-metadata.service.ts +++ b/server/src/metadata/object-metadata/services/object-metadata.service.ts @@ -131,7 +131,7 @@ export class ObjectMetadataService extends TypeOrmQueryService { ); } - public async deleteObjectsAndFieldsMetadata(workspaceId: string) { + public async deleteObjectsMetadata(workspaceId: string) { await this.objectMetadataRepository.delete({ workspaceId }); } } diff --git a/server/src/metadata/tenant-initialisation/tenant-initialisation.module.ts b/server/src/metadata/tenant-initialisation/tenant-initialisation.module.ts index af4e57ced..07d8d0f73 100644 --- a/server/src/metadata/tenant-initialisation/tenant-initialisation.module.ts +++ b/server/src/metadata/tenant-initialisation/tenant-initialisation.module.ts @@ -5,6 +5,7 @@ import { MigrationRunnerModule } from 'src/metadata/migration-runner/migration-r import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module'; import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module'; import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module'; +import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module'; import { TenantInitialisationService } from './tenant-initialisation.service'; @@ -14,6 +15,7 @@ import { TenantInitialisationService } from './tenant-initialisation.service'; TenantMigrationModule, MigrationRunnerModule, ObjectMetadataModule, + FieldMetadataModule, DataSourceMetadataModule, ], exports: [TenantInitialisationService], diff --git a/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts b/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts index c291f6f4a..31154e52b 100644 --- a/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts +++ b/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts @@ -6,6 +6,7 @@ import { DataSourceService } from 'src/metadata/data-source/data-source.service' import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service'; import { ObjectMetadataService } from 'src/metadata/object-metadata/services/object-metadata.service'; import { DataSourceMetadata } from 'src/metadata/data-source-metadata/data-source-metadata.entity'; +import { FieldMetadataService } from 'src/metadata/field-metadata/services/field-metadata.service'; import { standardObjectsPrefillData } from './standard-objects-prefill-data/standard-objects-prefill-data'; @@ -16,6 +17,7 @@ export class TenantInitialisationService { private readonly tenantMigrationService: TenantMigrationService, private readonly migrationRunnerService: MigrationRunnerService, private readonly objectMetadataService: ObjectMetadataService, + private readonly fieldMetadataService: FieldMetadataService, private readonly dataSourceMetadataService: DataSourceMetadataService, ) {} @@ -78,9 +80,8 @@ export class TenantInitialisationService { public async delete(workspaceId: string): Promise { // Delete data from metadata tables - await this.objectMetadataService.deleteObjectsAndFieldsMetadata( - workspaceId, - ); + await this.fieldMetadataService.deleteFieldsMetadata(workspaceId); + await this.objectMetadataService.deleteObjectsMetadata(workspaceId); await this.tenantMigrationService.delete(workspaceId); await this.dataSourceMetadataService.delete(workspaceId); // Delete schema