From 462c7ebdc15ba66989666e04e3aac7b3379737a1 Mon Sep 17 00:00:00 2001 From: martmull Date: Tue, 7 Nov 2023 11:10:14 +0100 Subject: [PATCH] 0.2.0 cleaning script (#2379) * Move question to questions folder * Aggregate update result functions * Use lodash to compare list of objects * Remove favorites from tables * Add a workspace parameter * Move question after result log * Improve logging * Code review returns * Add only lodash.isequal --- server/package.json | 2 + .../clean-inactive-workspaces.command.ts | 80 ++++++++++++------- .../commands/database-command.module.ts | 2 +- .../{ => questions}/confirmation.question.ts | 0 server/src/utils/equal.ts | 13 --- server/yarn.lock | 21 +++-- 6 files changed, 69 insertions(+), 49 deletions(-) rename server/src/database/commands/{ => questions}/confirmation.question.ts (100%) delete mode 100644 server/src/utils/equal.ts diff --git a/server/package.json b/server/package.json index ddacb30c3..76a88ff9b 100644 --- a/server/package.json +++ b/server/package.json @@ -87,6 +87,7 @@ "jsonwebtoken": "^9.0.0", "lodash.camelcase": "^4.3.0", "lodash.isempty": "^4.4.0", + "lodash.isequal": "^4.5.0", "lodash.isobject": "^3.0.2", "lodash.kebabcase": "^4.1.1", "lodash.merge": "^4.6.2", @@ -123,6 +124,7 @@ "@types/graphql-upload": "^8.0.12", "@types/jest": "28.1.8", "@types/lodash.isempty": "^4.4.7", + "@types/lodash.isequal": "^4.5.7", "@types/lodash.isobject": "^3.0.7", "@types/lodash.kebabcase": "^4.1.7", "@types/lodash.snakecase": "^4.1.7", diff --git a/server/src/database/commands/clean-inactive-workspaces.command.ts b/server/src/database/commands/clean-inactive-workspaces.command.ts index 3f0ddce1d..e013d0e6f 100644 --- a/server/src/database/commands/clean-inactive-workspaces.command.ts +++ b/server/src/database/commands/clean-inactive-workspaces.command.ts @@ -4,13 +4,13 @@ import { InquirerService, Option, } from 'nest-commander'; +import isEqual from 'lodash.isequal'; import { PrismaService } from 'src/database/prisma.service'; import peopleSeed from 'src/core/person/seed-data/people.json'; import companiesSeed from 'src/core/company/seed-data/companies.json'; import pipelineStagesSeed from 'src/core/pipeline/seed-data/pipeline-stages.json'; import pipelinesSeed from 'src/core/pipeline/seed-data/sales-pipeline.json'; -import { arraysEqual } from 'src/utils/equal'; import { WorkspaceService } from 'src/core/workspace/services/workspace.service'; interface DataCleanInactiveOptions { @@ -18,6 +18,7 @@ interface DataCleanInactiveOptions { sameAsSeedDays?: number; dryRun?: boolean; confirmation?: boolean; + workspaceId?: string; } interface ActivityReport { @@ -48,6 +49,14 @@ export class DataCleanInactiveCommand extends CommandRunner { super(); } + @Option({ + flags: '-w, --workspaceId [workspace id]', + description: 'Specific workspaceId to apply cleaning', + }) + parseWorkspace(val: string): string { + return val; + } + @Option({ flags: '-d, --days [inactive days threshold]', description: 'Inactive days threshold', @@ -82,20 +91,20 @@ export class DataCleanInactiveCommand extends CommandRunner { !name.startsWith('$') && !name.includes('user') && !name.includes('refreshToken') && - !name.includes('workspace'), + !name.includes('workspace') && + !name.includes('favorite'), ); } async getTableMaxUpdatedAt(table, workspace) { - try { - return await this.prismaService.client[table].aggregate({ - _max: { updatedAt: true }, - where: { workspaceId: { equals: workspace.id } }, - }); - } catch (e) {} + return await this.prismaService.client[table].aggregate({ + _max: { updatedAt: true }, + where: { workspaceId: { equals: workspace.id } }, + }); } - updateResult(result, workspace, newUpdatedAt) { + async addMaxUpdatedAtToWorkspaces(result, workspace, table) { + const newUpdatedAt = await this.getTableMaxUpdatedAt(table, workspace); if (!result.activityReport[workspace.id]) { result.activityReport[workspace.id] = { displayName: workspace.displayName, @@ -147,10 +156,10 @@ export class DataCleanInactiveCommand extends CommandRunner { where: { workspaceId: { equals: workspace.id } }, }); if ( - arraysEqual(people, peopleSeed) && - arraysEqual(companies, companiesSeed) && - arraysEqual(pipelineStages, pipelineStagesSeed) && - arraysEqual(pipelines, [pipelinesSeed]) + isEqual(people, peopleSeed) && + isEqual(companies, companiesSeed) && + isEqual(pipelineStages, pipelineStagesSeed) && + isEqual(pipelines, [pipelinesSeed]) ) { result.sameAsSeedWorkspaces[workspace.id] = { displayName: workspace.displayName, @@ -158,14 +167,18 @@ export class DataCleanInactiveCommand extends CommandRunner { } } - async findInactiveWorkspaces(result) { - const workspaces = await this.prismaService.client.workspace.findMany(); + async findInactiveWorkspaces(result, options) { + const where = options.workspaceId + ? { id: { equals: options.workspaceId } } + : {}; + const workspaces = await this.prismaService.client.workspace.findMany({ + where, + }); const tables = this.getRelevantTables(); for (const workspace of workspaces) { await this.detectWorkspacesWithSeedDataOnly(result, workspace); for (const table of tables) { - const maxUpdatedAt = await this.getTableMaxUpdatedAt(table, workspace); - this.updateResult(result, workspace, maxUpdatedAt); + await this.addMaxUpdatedAtToWorkspaces(result, workspace, table); } } } @@ -195,26 +208,47 @@ export class DataCleanInactiveCommand extends CommandRunner { console.log('Deleting inactive workspaces'); } for (const workspaceId in result.activityReport) { + process.stdout.write(`- deleting ${workspaceId} ...`); await this.workspaceService.deleteWorkspace({ workspaceId, }); - console.log(`- ${workspaceId} deleted`); + console.log(' done!'); } if (Object.keys(result.sameAsSeedWorkspaces).length) { console.log('Deleting same as Seed workspaces'); } for (const workspaceId in result.sameAsSeedWorkspaces) { + process.stdout.write(`- deleting ${workspaceId} ...`); await this.workspaceService.deleteWorkspace({ workspaceId, }); - console.log(`- ${workspaceId} deleted`); + console.log(' done!'); } } + displayResults(result) { + const workspacesToDelete = new Set(); + for (const workspaceId in result.activityReport) { + workspacesToDelete.add(workspaceId); + } + for (const workspaceId in result.sameAsSeedWorkspaces) { + workspacesToDelete.add(workspaceId); + } + console.log(`${workspacesToDelete.size} workspace(s) will be deleted:`); + console.log(result); + } + async run( _passedParam: string[], options: DataCleanInactiveOptions, ): Promise { + const result: DataCleanResults = { + activityReport: {}, + sameAsSeedWorkspaces: {}, + }; + await this.findInactiveWorkspaces(result, options); + this.filterResults(result, options); + this.displayResults(result); if (!options.dryRun) { options = await this.inquiererService.ask('confirm', options); if (!options.confirmation) { @@ -222,16 +256,8 @@ export class DataCleanInactiveCommand extends CommandRunner { return; } } - const result: DataCleanResults = { - activityReport: {}, - sameAsSeedWorkspaces: {}, - }; - await this.findInactiveWorkspaces(result); - this.filterResults(result, options); if (!options.dryRun) { await this.delete(result); - } else { - console.log(result); } } } diff --git a/server/src/database/commands/database-command.module.ts b/server/src/database/commands/database-command.module.ts index 8e2ca1eec..3d89ee29e 100644 --- a/server/src/database/commands/database-command.module.ts +++ b/server/src/database/commands/database-command.module.ts @@ -1,7 +1,7 @@ import { Module } from '@nestjs/common'; import { DataCleanInactiveCommand } from 'src/database/commands/clean-inactive-workspaces.command'; -import { ConfirmationQuestion } from 'src/database/commands/confirmation.question'; +import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question'; import { WorkspaceService } from 'src/core/workspace/services/workspace.service'; import { PipelineModule } from 'src/core/pipeline/pipeline.module'; import { CompanyModule } from 'src/core/company/company.module'; diff --git a/server/src/database/commands/confirmation.question.ts b/server/src/database/commands/questions/confirmation.question.ts similarity index 100% rename from server/src/database/commands/confirmation.question.ts rename to server/src/database/commands/questions/confirmation.question.ts diff --git a/server/src/utils/equal.ts b/server/src/utils/equal.ts deleted file mode 100644 index a717620df..000000000 --- a/server/src/utils/equal.ts +++ /dev/null @@ -1,13 +0,0 @@ -//https://stackoverflow.com/questions/27030/comparing-arrays-of-objects-in-javascript -export const objectsEqual = (o1, o2) => { - return ( - Object.keys(o1).length === Object.keys(o2).length && - Object.keys(o1).every((p) => o1[p] === o2[p]) - ); -}; - -export const arraysEqual = (a1, a2) => { - return ( - a1.length === a2.length && a1.every((o, idx) => objectsEqual(o, a2[idx])) - ); -}; diff --git a/server/yarn.lock b/server/yarn.lock index 2c41d81ea..9706ae9c1 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -3056,6 +3056,13 @@ dependencies: "@types/lodash" "*" +"@types/lodash.isequal@^4.5.7": + version "4.5.7" + resolved "https://registry.yarnpkg.com/@types/lodash.isequal/-/lodash.isequal-4.5.7.tgz#8d95603728dc7a8070c37d12d98b27b4d665849a" + integrity sha512-UJQsb7aW8JU/h3fivQXVRDp9EKi98T9iQcVeTXBxcD4jApgGgbrET/0hVS6vH/YoYpqkcToMU5fSNPEiWVZgDg== + dependencies: + "@types/lodash" "*" + "@types/lodash.isobject@^3.0.7": version "3.0.7" resolved "https://registry.yarnpkg.com/@types/lodash.isobject/-/lodash.isobject-3.0.7.tgz#8a37beea56512f0ae86f8d48ea01e2ea9b79c185" @@ -5885,20 +5892,13 @@ graphql-subscriptions@2.0.0: dependencies: iterall "^1.3.0" -graphql-tag@2.12.6, graphql-tag@^2.11.0: +graphql-tag@2.12.6, graphql-tag@^2.11.0, graphql-tag@^2.12.6: version "2.12.6" resolved "https://registry.npmjs.org/graphql-tag/-/graphql-tag-2.12.6.tgz" integrity sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg== dependencies: tslib "^2.1.0" -graphql-tag@^2.12.6: - version "2.12.6" - resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.6.tgz#d441a569c1d2537ef10ca3d1633b48725329b5f1" - integrity sha512-FdSNcu2QQcWnM2VNvSCCDCVS5PpPqpzgFT8+GXzqJuoDd0CBncxCY278u4mhRO7tMgo2JjgJA5aZ+nWSQ/Z+xg== - dependencies: - tslib "^2.1.0" - graphql-type-json@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/graphql-type-json/-/graphql-type-json-0.3.2.tgz#f53a851dbfe07bd1c8157d24150064baab41e115" @@ -7046,6 +7046,11 @@ lodash.isempty@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e" integrity sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg== +lodash.isequal@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" + integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== + lodash.isobject@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/lodash.isobject/-/lodash.isobject-3.0.2.tgz#3c8fb8d5b5bf4bf90ae06e14f2a530a4ed935e1d"