diff --git a/server/scripts/setup-db.ts b/server/scripts/setup-db.ts index 5edab5b07..d8845071f 100644 --- a/server/scripts/setup-db.ts +++ b/server/scripts/setup-db.ts @@ -1,28 +1,6 @@ -import { ConfigService } from '@nestjs/config'; - import console from 'console'; -import { config } from 'dotenv'; -import { DataSource } from 'typeorm'; - -config(); - -const configService = new ConfigService(); - -export const connectionSource = new DataSource({ - type: 'postgres', - logging: false, - url: configService.get('PG_DATABASE_URL'), -}); - -const performQuery = async (query: string, consoleDescription: string) => { - try { - await connectionSource.query(query); - console.log(`Performed '${consoleDescription}' successfully`); - } catch (err) { - console.error(`Failed to perform '${consoleDescription}':`, err); - } -}; +import { connectionSource, performQuery } from './utils'; connectionSource .initialize() diff --git a/server/scripts/truncate-db.ts b/server/scripts/truncate-db.ts index 8117f6a62..4c616ad98 100644 --- a/server/scripts/truncate-db.ts +++ b/server/scripts/truncate-db.ts @@ -1,28 +1,6 @@ -import { ConfigService } from '@nestjs/config'; - import console from 'console'; -import { config } from 'dotenv'; -import { DataSource } from 'typeorm'; - -config(); - -const configService = new ConfigService(); - -export const connectionSource = new DataSource({ - type: 'postgres', - logging: false, - url: configService.get('PG_DATABASE_URL'), -}); - -const performQuery = async (query: string, consoleDescription: string) => { - try { - await connectionSource.query(query); - console.log(`Performed '${consoleDescription}' successfully`); - } catch (err) { - console.error(`Failed to perform '${consoleDescription}':`, err); - } -}; +import { connectionSource, performQuery } from './utils'; connectionSource .initialize() diff --git a/server/scripts/utils.ts b/server/scripts/utils.ts new file mode 100644 index 000000000..63fb11946 --- /dev/null +++ b/server/scripts/utils.ts @@ -0,0 +1,28 @@ +import { ConfigService } from '@nestjs/config'; + +import console from 'console'; + +import { config } from 'dotenv'; +import { DataSource } from 'typeorm'; + +config(); +const configService = new ConfigService(); +export const connectionSource = new DataSource({ + type: 'postgres', + logging: false, + url: configService.get('PG_DATABASE_URL'), +}); + +export const performQuery = async ( + query: string, + consoleDescription: string, + withLog = true, +) => { + try { + const result = await connectionSource.query(query); + withLog && console.log(`Performed '${consoleDescription}' successfully`); + return result; + } catch (err) { + withLog && console.error(`Failed to perform '${consoleDescription}':`, err); + } +}; diff --git a/server/src/command.module.ts b/server/src/command.module.ts index 1bd641d2a..a875dbd6a 100644 --- a/server/src/command.module.ts +++ b/server/src/command.module.ts @@ -1,10 +1,12 @@ import { Module } from '@nestjs/common'; +import { DatabaseCommandModule } from 'src/database/commands/database-command.module'; + import { AppModule } from './app.module'; import { MetadataCommandModule } from './metadata/commands/metadata-command.module'; @Module({ - imports: [AppModule, MetadataCommandModule], + imports: [AppModule, MetadataCommandModule, DatabaseCommandModule], }) export class CommandModule {} diff --git a/server/src/core/user/user.service.ts b/server/src/core/user/user.service.ts index 1256e4ae8..d28d95509 100644 --- a/server/src/core/user/user.service.ts +++ b/server/src/core/user/user.service.ts @@ -137,8 +137,6 @@ export class UserService { // Delete entire workspace await this.workspaceService.deleteWorkspace({ workspaceId, - userId, - select: { id: true }, }); } else { await this.prismaService.client.$transaction([ diff --git a/server/src/core/workspace/resolvers/workspace.resolver.ts b/server/src/core/workspace/resolvers/workspace.resolver.ts index b2feb8d51..f1b42c0d9 100644 --- a/server/src/core/workspace/resolvers/workspace.resolver.ts +++ b/server/src/core/workspace/resolvers/workspace.resolver.ts @@ -24,8 +24,6 @@ import { UpdateWorkspaceAbilityHandler, DeleteWorkspaceAbilityHandler, } from 'src/ability/handlers/workspace.ability-handler'; -import { AuthUser } from 'src/decorators/auth-user.decorator'; -import { User } from 'src/core/@generated/user/user.model'; @UseGuards(JwtAuthGuard) @Resolver(() => Workspace) @@ -108,12 +106,10 @@ export class WorkspaceResolver { @AuthWorkspace() { id: workspaceId }: Workspace, @PrismaSelector({ modelName: 'Workspace' }) { value: select }: PrismaSelect<'Workspace'>, - @AuthUser() { id: userId }: User, ) { return this.workspaceService.deleteWorkspace({ workspaceId, select, - userId, }); } } diff --git a/server/src/core/workspace/services/workspace.service.ts b/server/src/core/workspace/services/workspace.service.ts index 372513e5a..84484ed83 100644 --- a/server/src/core/workspace/services/workspace.service.ts +++ b/server/src/core/workspace/services/workspace.service.ts @@ -93,12 +93,10 @@ export class WorkspaceService { async deleteWorkspace({ workspaceId, - select, - userId, + select = { id: true }, }: { workspaceId: string; - select: Prisma.WorkspaceSelect; - userId: string; + select?: Prisma.WorkspaceSelect; }) { const workspace = await this.findUnique({ where: { id: workspaceId }, @@ -109,19 +107,17 @@ export class WorkspaceService { const where = { workspaceId }; const { - user, workspaceMember, - refreshToken, attachment, comment, activityTarget, activity, + apiKey, + favorite, + webHook, } = this.prismaService.client; - const activitys = await activity.findMany({ - where: { authorId: userId }, - }); - + // We don't delete user or refresh tokens as they can belong to another workspace await this.prismaService.client.$transaction([ this.pipelineProgressService.deleteMany({ where, @@ -147,22 +143,20 @@ export class WorkspaceService { comment.deleteMany({ where, }), - ...activitys.map(({ id: activityId }) => - activityTarget.deleteMany({ - where: { activityId }, - }), - ), + activityTarget.deleteMany({ + where, + }), activity.deleteMany({ where, }), - refreshToken.deleteMany({ - where: { userId }, + apiKey.deleteMany({ + where, }), - // Todo delete all users from this workspace - user.delete({ - where: { - id: userId, - }, + favorite.deleteMany({ + where, + }), + webHook.deleteMany({ + where, }), this.delete({ where: { id: workspaceId } }), ]); diff --git a/server/src/database/commands/clean-inactive-workspaces.command.ts b/server/src/database/commands/clean-inactive-workspaces.command.ts new file mode 100644 index 000000000..3f0ddce1d --- /dev/null +++ b/server/src/database/commands/clean-inactive-workspaces.command.ts @@ -0,0 +1,237 @@ +import { + Command, + CommandRunner, + InquirerService, + Option, +} from 'nest-commander'; + +import { PrismaService } from 'src/database/prisma.service'; +import peopleSeed from 'src/core/person/seed-data/people.json'; +import companiesSeed from 'src/core/company/seed-data/companies.json'; +import pipelineStagesSeed from 'src/core/pipeline/seed-data/pipeline-stages.json'; +import pipelinesSeed from 'src/core/pipeline/seed-data/sales-pipeline.json'; +import { arraysEqual } from 'src/utils/equal'; +import { WorkspaceService } from 'src/core/workspace/services/workspace.service'; + +interface DataCleanInactiveOptions { + days?: number; + sameAsSeedDays?: number; + dryRun?: boolean; + confirmation?: boolean; +} + +interface ActivityReport { + displayName: string; + maxUpdatedAt: string; + inactiveDays: number; +} + +interface SameAsSeedWorkspace { + displayName: string; +} + +interface DataCleanResults { + activityReport: { [key: string]: ActivityReport }; + sameAsSeedWorkspaces: { [key: string]: SameAsSeedWorkspace }; +} + +@Command({ + name: 'workspaces:clean-inactive', + description: 'Clean inactive workspaces from the public database schema', +}) +export class DataCleanInactiveCommand extends CommandRunner { + constructor( + private readonly prismaService: PrismaService, + private readonly workspaceService: WorkspaceService, + private readonly inquiererService: InquirerService, + ) { + super(); + } + + @Option({ + flags: '-d, --days [inactive days threshold]', + description: 'Inactive days threshold', + defaultValue: 60, + }) + parseDays(val: string): number { + return Number(val); + } + + @Option({ + flags: '-s, --same-as-seed-days [same as seed days threshold]', + description: 'Same as seed days threshold', + defaultValue: 10, + }) + parseSameAsSeedDays(val: string): number { + return Number(val); + } + + @Option({ + flags: '--dry-run [dry run]', + description: 'List inactive workspaces without removing them', + }) + parseDryRun(val: string): boolean { + return Boolean(val); + } + + // We look for public tables which contains workspaceId and updatedAt columns + getRelevantTables() { + return Object.keys(this.prismaService.client).filter( + (name) => + !name.startsWith('_') && + !name.startsWith('$') && + !name.includes('user') && + !name.includes('refreshToken') && + !name.includes('workspace'), + ); + } + + async getTableMaxUpdatedAt(table, workspace) { + try { + return await this.prismaService.client[table].aggregate({ + _max: { updatedAt: true }, + where: { workspaceId: { equals: workspace.id } }, + }); + } catch (e) {} + } + + updateResult(result, workspace, newUpdatedAt) { + if (!result.activityReport[workspace.id]) { + result.activityReport[workspace.id] = { + displayName: workspace.displayName, + maxUpdatedAt: null, + }; + } + if ( + newUpdatedAt && + newUpdatedAt._max.updatedAt && + new Date(result.activityReport[workspace.id].maxUpdatedAt) < + new Date(newUpdatedAt._max.updatedAt) + ) { + result.activityReport[workspace.id].maxUpdatedAt = + newUpdatedAt._max.updatedAt; + } + } + + async detectWorkspacesWithSeedDataOnly(result, workspace) { + const companies = await this.prismaService.client.company.findMany({ + select: { name: true, domainName: true, address: true, employees: true }, + where: { workspaceId: { equals: workspace.id } }, + }); + const people = await this.prismaService.client.person.findMany({ + select: { + firstName: true, + lastName: true, + city: true, + email: true, + avatarUrl: true, + }, + where: { workspaceId: { equals: workspace.id } }, + }); + const pipelineStages = + await this.prismaService.client.pipelineStage.findMany({ + select: { + name: true, + color: true, + position: true, + type: true, + }, + where: { workspaceId: { equals: workspace.id } }, + }); + const pipelines = await this.prismaService.client.pipeline.findMany({ + select: { + name: true, + icon: true, + pipelineProgressableType: true, + }, + where: { workspaceId: { equals: workspace.id } }, + }); + if ( + arraysEqual(people, peopleSeed) && + arraysEqual(companies, companiesSeed) && + arraysEqual(pipelineStages, pipelineStagesSeed) && + arraysEqual(pipelines, [pipelinesSeed]) + ) { + result.sameAsSeedWorkspaces[workspace.id] = { + displayName: workspace.displayName, + }; + } + } + + async findInactiveWorkspaces(result) { + const workspaces = await this.prismaService.client.workspace.findMany(); + const tables = this.getRelevantTables(); + for (const workspace of workspaces) { + await this.detectWorkspacesWithSeedDataOnly(result, workspace); + for (const table of tables) { + const maxUpdatedAt = await this.getTableMaxUpdatedAt(table, workspace); + this.updateResult(result, workspace, maxUpdatedAt); + } + } + } + + filterResults(result, options) { + for (const workspaceId in result.activityReport) { + const timeDifferenceInSeconds = Math.abs( + new Date().getTime() - + new Date(result.activityReport[workspaceId].maxUpdatedAt).getTime(), + ); + const timeDifferenceInDays = Math.ceil( + timeDifferenceInSeconds / (1000 * 3600 * 24), + ); + if (timeDifferenceInDays < options.sameAsSeedDays) { + delete result.sameAsSeedWorkspaces[workspaceId]; + } + if (timeDifferenceInDays < options.days) { + delete result.activityReport[workspaceId]; + } else { + result.activityReport[workspaceId].inactiveDays = timeDifferenceInDays; + } + } + } + + async delete(result) { + if (Object.keys(result.activityReport).length) { + console.log('Deleting inactive workspaces'); + } + for (const workspaceId in result.activityReport) { + await this.workspaceService.deleteWorkspace({ + workspaceId, + }); + console.log(`- ${workspaceId} deleted`); + } + if (Object.keys(result.sameAsSeedWorkspaces).length) { + console.log('Deleting same as Seed workspaces'); + } + for (const workspaceId in result.sameAsSeedWorkspaces) { + await this.workspaceService.deleteWorkspace({ + workspaceId, + }); + console.log(`- ${workspaceId} deleted`); + } + } + + async run( + _passedParam: string[], + options: DataCleanInactiveOptions, + ): Promise { + if (!options.dryRun) { + options = await this.inquiererService.ask('confirm', options); + if (!options.confirmation) { + console.log('Cleaning aborted'); + return; + } + } + const result: DataCleanResults = { + activityReport: {}, + sameAsSeedWorkspaces: {}, + }; + await this.findInactiveWorkspaces(result); + this.filterResults(result, options); + if (!options.dryRun) { + await this.delete(result); + } else { + console.log(result); + } + } +} diff --git a/server/src/database/commands/confirmation.question.ts b/server/src/database/commands/confirmation.question.ts new file mode 100644 index 000000000..f6769ba0c --- /dev/null +++ b/server/src/database/commands/confirmation.question.ts @@ -0,0 +1,16 @@ +import { Question, QuestionSet } from 'nest-commander'; + +@QuestionSet({ + name: 'confirm', +}) +export class ConfirmationQuestion { + @Question({ + type: 'confirm', + name: 'confirmation', + message: + "You are about to delete data from database. Are you sure to continue? Consider the '--dry-run' option first", + }) + parseConfirm(val: string): boolean { + return Boolean(val); + } +} diff --git a/server/src/database/commands/database-command.module.ts b/server/src/database/commands/database-command.module.ts new file mode 100644 index 000000000..8e2ca1eec --- /dev/null +++ b/server/src/database/commands/database-command.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; + +import { DataCleanInactiveCommand } from 'src/database/commands/clean-inactive-workspaces.command'; +import { ConfirmationQuestion } from 'src/database/commands/confirmation.question'; +import { WorkspaceService } from 'src/core/workspace/services/workspace.service'; +import { PipelineModule } from 'src/core/pipeline/pipeline.module'; +import { CompanyModule } from 'src/core/company/company.module'; +import { PersonModule } from 'src/core/person/person.module'; +import { TenantInitialisationModule } from 'src/metadata/tenant-initialisation/tenant-initialisation.module'; +import { PrismaModule } from 'src/database/prisma.module'; + +@Module({ + imports: [ + PipelineModule, + CompanyModule, + PersonModule, + TenantInitialisationModule, + PrismaModule, + ], + providers: [DataCleanInactiveCommand, ConfirmationQuestion, WorkspaceService], +}) +export class DatabaseCommandModule {} diff --git a/server/src/utils/equal.ts b/server/src/utils/equal.ts new file mode 100644 index 000000000..a717620df --- /dev/null +++ b/server/src/utils/equal.ts @@ -0,0 +1,13 @@ +//https://stackoverflow.com/questions/27030/comparing-arrays-of-objects-in-javascript +export const objectsEqual = (o1, o2) => { + return ( + Object.keys(o1).length === Object.keys(o2).length && + Object.keys(o1).every((p) => o1[p] === o2[p]) + ); +}; + +export const arraysEqual = (a1, a2) => { + return ( + a1.length === a2.length && a1.every((o, idx) => objectsEqual(o, a2[idx])) + ); +};