From efc43208d3e53cab4b6323552b289adb7474a773 Mon Sep 17 00:00:00 2001 From: Etienne <45695613+etiennejouan@users.noreply.github.com> Date: Mon, 19 May 2025 14:45:15 +0200 Subject: [PATCH] add command to clean not found files (#12094) closes https://github.com/twentyhq/core-team-issues/issues/883 tested on person, workspaceMember, workspace and attachments files - dry/normal --- .../0-54-clean-not-found-files.command.ts | 225 ++++++++++++++++++ .../0-54-upgrade-version-command.module.ts | 7 +- 2 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-clean-not-found-files.command.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-clean-not-found-files.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-clean-not-found-files.command.ts new file mode 100644 index 000000000..cf31615b8 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-clean-not-found-files.command.ts @@ -0,0 +1,225 @@ +import { InjectRepository } from '@nestjs/typeorm'; + +import { basename, dirname } from 'path'; + +import { isNonEmptyString } from '@sniptt/guards'; +import { Command } from 'nest-commander'; +import { Equal, Not, Repository } from 'typeorm'; + +import { + FileStorageException, + FileStorageExceptionCode, +} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception'; + +import { + ActiveOrSuspendedWorkspacesMigrationCommandRunner, + RunOnWorkspaceArgs, +} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner'; +import { FileService } from 'src/engine/core-modules/file/services/file.service'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; +import { AttachmentWorkspaceEntity } from 'src/modules/attachment/standard-objects/attachment.workspace-entity'; +import { PersonWorkspaceEntity } from 'src/modules/person/standard-objects/person.workspace-entity'; +import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; + +@Command({ + name: 'upgrade:0-54:clean-not-found-files', + description: 'Clean not found files', +}) +export class CleanNotFoundFilesCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner { + constructor( + @InjectRepository(Workspace, 'core') + protected readonly workspaceRepository: Repository, + protected readonly twentyORMGlobalManager: TwentyORMGlobalManager, + private readonly fileService: FileService, + ) { + super(workspaceRepository, twentyORMGlobalManager); + } + + override async runOnWorkspace({ + index, + total, + workspaceId, + options, + }: RunOnWorkspaceArgs): Promise { + this.logger.log( + `Running command for workspace ${workspaceId} ${index + 1}/${total}`, + ); + + await this.cleanNotFoundFiles(workspaceId, !!options.dryRun); + } + + private async cleanNotFoundFiles(workspaceId: string, dryRun: boolean) { + await this.cleanWorkspaceLogo(workspaceId, dryRun); + await this.softDeleteAttachments(workspaceId, dryRun); + await this.cleanWorkspaceMembersAvatarUrl(workspaceId, dryRun); + await this.cleanPeopleAvatarUrl(workspaceId, dryRun); + } + + private async checkIfFileIsFound(path: string, workspaceId: string) { + if (path.startsWith('https://')) return true; // seed data + + try { + await this.fileService.getFileStream( + dirname(path), + basename(path), + workspaceId, + ); + } catch (error) { + if ( + error instanceof FileStorageException && + error.code === FileStorageExceptionCode.FILE_NOT_FOUND + ) { + return false; + } + } + + return true; + } + + private async cleanWorkspaceLogo(workspaceId: string, dryRun: boolean) { + const workspace = await this.workspaceRepository.findOneOrFail({ + where: { + id: workspaceId, + }, + }); + + if (!isNonEmptyString(workspace.logo)) return; + + const isFileFound = await this.checkIfFileIsFound( + workspace.logo, + workspace.id, + ); + + if (isFileFound) return; + + if (!dryRun) + await this.workspaceRepository.update(workspace.id, { + logo: '', + }); + + this.logger.log( + `${dryRun ? 'Dry run - ' : ''}Set logo to '' for workspace ${workspace.id}`, + ); + } + + private async softDeleteAttachments(workspaceId: string, dryRun: boolean) { + const attachmentRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'attachment', + ); + const attachmentsCount = await attachmentRepository.count(); + const chunkSize = 10; + + const attachmentIdsToSoftDelete: string[] = []; + + for (let offset = 0; offset < attachmentsCount; offset += chunkSize) { + const attachmentsChunk = await attachmentRepository.find({ + skip: offset, + take: chunkSize, + }); + + const attachmentIdsToSoftDeleteChunk = await Promise.all( + attachmentsChunk.map(async (attachment) => { + const isFileFound = await this.checkIfFileIsFound( + attachment.fullPath, + workspaceId, + ); + + return isFileFound ? '' : attachment.id; + }), + ); + + attachmentIdsToSoftDelete.push( + ...attachmentIdsToSoftDeleteChunk.filter(isNonEmptyString), + ); + } + + if (attachmentIdsToSoftDelete.length === 0) return; + + if (!dryRun) + await attachmentRepository.softDelete(attachmentIdsToSoftDelete); + + this.logger.log( + `${dryRun ? 'Dry run - ' : ''}Deleted attachments ${attachmentIdsToSoftDelete.join(', ')}`, + ); + } + + private async cleanWorkspaceMembersAvatarUrl( + workspaceId: string, + dryRun: boolean, + ) { + const workspaceMemberRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'workspaceMember', + ); + const workspaceMembers = await workspaceMemberRepository.find({ + where: { + avatarUrl: Not(Equal('')), + }, + }); + + const workspaceMemberIdsToUpdate: string[] = []; + + for (const workspaceMember of workspaceMembers) { + const isFileFound = await this.checkIfFileIsFound( + workspaceMember.avatarUrl, + workspaceId, + ); + + if (isFileFound) continue; + + workspaceMemberIdsToUpdate.push(workspaceMember.id); + } + + if (workspaceMemberIdsToUpdate.length === 0) return; + + if (!dryRun) + await workspaceMemberRepository.update(workspaceMemberIdsToUpdate, { + avatarUrl: '', + }); + + this.logger.log( + `${dryRun ? 'Dry run - ' : ''}Set avatarUrl to '' for workspaceMembers ${workspaceMemberIdsToUpdate.join(', ')}`, + ); + } + + private async cleanPeopleAvatarUrl(workspaceId: string, dryRun: boolean) { + const personRepository = + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, + 'person', + ); + const people = await personRepository.find({ + where: { + avatarUrl: Not(Equal('')), + }, + }); + + const personIdsToUpdate: string[] = []; + + for (const person of people) { + const isFileFound = await this.checkIfFileIsFound( + person.avatarUrl, + workspaceId, + ); + + if (!isFileFound) { + personIdsToUpdate.push(person.id); + } + } + + if (personIdsToUpdate.length === 0) return; + + if (!dryRun) + await personRepository.update(personIdsToUpdate, { + avatarUrl: '', + }); + + this.logger.log( + `${dryRun ? 'Dry run - ' : ''}Set avatarUrl to '' for people ${personIdsToUpdate.join(', ')}`, + ); + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-upgrade-version-command.module.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-upgrade-version-command.module.ts index d9a015e7d..e205a792b 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-upgrade-version-command.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version-command/0-54/0-54-upgrade-version-command.module.ts @@ -1,14 +1,16 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { CleanNotFoundFilesCommand } from 'src/database/commands/upgrade-version-command/0-54/0-54-clean-not-found-files.command'; +import { FixCreatedByDefaultValueCommand } from 'src/database/commands/upgrade-version-command/0-54/0-54-created-by-default-value.command'; import { FixStandardSelectFieldsPositionCommand } from 'src/database/commands/upgrade-version-command/0-54/0-54-fix-standard-select-fields-position.command'; +import { FileModule } from 'src/engine/core-modules/file/file.module'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; -import { FixCreatedByDefaultValueCommand } from 'src/database/commands/upgrade-version-command/0-54/0-54-created-by-default-value.command'; @Module({ imports: [ @@ -20,14 +22,17 @@ import { FixCreatedByDefaultValueCommand } from 'src/database/commands/upgrade-v WorkspaceDataSourceModule, WorkspaceMigrationRunnerModule, WorkspaceMetadataVersionModule, + FileModule, ], providers: [ FixStandardSelectFieldsPositionCommand, FixCreatedByDefaultValueCommand, + CleanNotFoundFilesCommand, ], exports: [ FixStandardSelectFieldsPositionCommand, FixCreatedByDefaultValueCommand, + CleanNotFoundFilesCommand, ], }) export class V0_54_UpgradeVersionCommandModule {}