From a424c63476140fdb72dad937c56a504ab61189c2 Mon Sep 17 00:00:00 2001 From: rostaklein Date: Thu, 1 Aug 2024 18:07:22 +0200 Subject: [PATCH] file storage workspace id prefix (#6230) closes https://github.com/twentyhq/twenty/issues/6155 just an idea, i guess this could work well, but im open for discussion --------- Co-authored-by: Weiko --- ...23-update-file-folder-structure.command.ts | 230 ++++++++++++++++++ .../0-23/0-23-upgrade-version.command.ts | 13 +- .../0-23/0-23-upgrade-version.module.ts | 7 + .../workspace-query-runner/factories/index.ts | 5 +- .../factories/query-result-getters.factory.ts | 75 ------ .../attachment-query-result-getter.handler.ts | 45 ++++ .../query-result-getter-handler.interface.ts | 3 + .../query-result-getters.factory.ts | 73 ++++++ .../workspace-query-runner.service.ts | 14 +- .../auth/services/sign-in-up.service.ts | 31 ++- .../file/controllers/file.controller.ts | 50 ++-- .../resolvers/file-upload.resolver.ts | 12 +- .../services/file-upload.service.ts | 26 +- .../core-modules/file/file.utils.spec.ts | 61 +++++ .../engine/core-modules/file/file.utils.ts | 2 +- .../file/guards/file-path-guard.ts | 37 +-- .../resolvers/file-upload.resolver.spec.ts | 27 -- .../file/resolvers/file-upload.resolver.ts | 57 ----- .../file/services/file.service.ts | 37 ++- .../engine/core-modules/user/user.resolver.ts | 4 +- .../workspace/workspace.resolver.ts | 1 + .../interfaces/storage-driver.interface.ts | 4 + .../file-storage/drivers/local.driver.ts | 54 +++- .../file-storage/drivers/s3.driver.ts | 71 +++++- .../file-storage/file-storage.service.ts | 7 + .../interfaces/file-storage-exception.ts | 12 + 26 files changed, 727 insertions(+), 231 deletions(-) create mode 100644 packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command.ts delete mode 100644 packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters.factory.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/attachment-query-result-getter.handler.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory.ts create mode 100644 packages/twenty-server/src/engine/core-modules/file/file.utils.spec.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.spec.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.ts create mode 100644 packages/twenty-server/src/engine/integrations/file-storage/interfaces/file-storage-exception.ts diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command.ts new file mode 100644 index 000000000..be890ed77 --- /dev/null +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command.ts @@ -0,0 +1,230 @@ +import { Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import chalk from 'chalk'; +import { Command, CommandRunner, Option } from 'nest-commander'; +import pLimit from 'p-limit'; +import { Like, Repository } from 'typeorm'; + +import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface'; +import { + FileStorageException, + FileStorageExceptionCode, +} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception'; + +import { TypeORMService } from 'src/database/typeorm/typeorm.service'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; + +interface UpdateFileFolderStructureCommandOptions { + workspaceId?: string; +} + +@Command({ + name: 'upgrade-0-23:update-file-folder-structure', + description: 'Update file folder structure (prefixed per workspace)', +}) +export class UpdateFileFolderStructureCommand extends CommandRunner { + private readonly logger = new Logger(UpdateFileFolderStructureCommand.name); + constructor( + @InjectRepository(Workspace, 'core') + private readonly workspaceRepository: Repository, + private readonly typeORMService: TypeORMService, + private readonly dataSourceService: DataSourceService, + private readonly fileStorageService: FileStorageService, + ) { + super(); + } + + @Option({ + flags: '-w, --workspace-id [workspace_id]', + description: 'workspace id. Command runs on all workspaces if not provided', + required: false, + }) + parseWorkspaceId(value: string): string { + return value; + } + + async run( + _passedParam: string[], + options: UpdateFileFolderStructureCommandOptions, + ): Promise { + const workspaceIds = options.workspaceId + ? [options.workspaceId] + : (await this.workspaceRepository.find()).map( + (workspace) => workspace.id, + ); + + if (!workspaceIds.length) { + this.logger.log(chalk.yellow('No workspace found')); + + return; + } + + this.logger.log( + chalk.green(`Running command on ${workspaceIds.length} workspaces`), + ); + + for (const workspaceId of workspaceIds) { + const dataSourceMetadata = + await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId( + workspaceId, + ); + + if (!dataSourceMetadata) { + this.logger.log( + `Could not find dataSourceMetadata for workspace ${workspaceId}`, + ); + continue; + } + + const workspaceDataSource = + await this.typeORMService.connectToDataSource(dataSourceMetadata); + + if (!workspaceDataSource) { + throw new Error( + `Could not connect to dataSource for workspace ${workspaceId}`, + ); + } + + const workspaceQueryRunner = workspaceDataSource.createQueryRunner(); + + const attachmentsToMove = (await workspaceQueryRunner.query( + `SELECT id, "fullPath" FROM "${dataSourceMetadata.schema}"."attachment" WHERE "fullPath" LIKE '${FileFolder.Attachment}/%'`, + )) as { id: string; fullPath: string }[]; + + const workspaceMemberAvatarsToMove = (await workspaceQueryRunner.query( + `SELECT id, "avatarUrl" as "fullPath" FROM "${dataSourceMetadata.schema}"."workspaceMember" WHERE "avatarUrl" LIKE '${FileFolder.ProfilePicture}/%'`, + )) as { id: string; fullPath: string }[]; + + const personAvatarsToMove = (await workspaceQueryRunner.query( + `SELECT id, "avatarUrl" as "fullPath" FROM "${dataSourceMetadata.schema}"."person" WHERE "avatarUrl" LIKE '${FileFolder.PersonPicture}/%'`, + )) as { id: string; fullPath: string }[]; + + const workspacePictureToMove = await this.workspaceRepository.findOneBy({ + id: workspaceId, + logo: Like(`${FileFolder.WorkspaceLogo}/%`), + }); + + try { + const updatedAttachments = await this.moveFiles( + workspaceId, + attachmentsToMove, + ); + + this.logger.log( + chalk.green( + `Moved ${updatedAttachments.length} attachments in workspace ${workspaceId}`, + ), + ); + } catch (e) { + this.logger.error(e); + } + + try { + const updatedWorkspaceMemberAvatars = await this.moveFiles( + workspaceId, + workspaceMemberAvatarsToMove, + ); + + this.logger.log( + chalk.green( + `Moved ${updatedWorkspaceMemberAvatars.length} workspaceMemberAvatars in workspace ${workspaceId}`, + ), + ); + } catch (e) { + this.logger.error(e); + } + + try { + const updatedPersonAvatars = await this.moveFiles( + workspaceId, + personAvatarsToMove, + ); + + this.logger.log( + chalk.green( + `Moved ${updatedPersonAvatars.length} personAvatars in workspace ${workspaceId}`, + ), + ); + } catch (e) { + this.logger.error(e); + } + + if (workspacePictureToMove?.logo) { + await this.moveFiles(workspaceId, [ + { + id: workspacePictureToMove.id, + fullPath: workspacePictureToMove.logo, + }, + ]); + + this.logger.log( + chalk.green(`Moved workspacePicture in workspace ${workspaceId}`), + ); + } + + this.logger.log( + chalk.green(`Running command on workspace ${workspaceId} done`), + ); + } + + this.logger.log(chalk.green(`Command completed!`)); + } + + private async moveFiles( + workspaceId: string, + filesToMove: { id: string; fullPath: string }[], + ): Promise> { + const batchSize = 20; + const limit = pLimit(batchSize); + + const moveFile = async ({ + id, + fullPath, + }: { + id: string; + fullPath: string; + }) => { + const pathParts = fullPath.split('/'); + const filename = pathParts.pop(); + + if (!filename) { + throw new Error(`Filename is empty for file ID: ${id}`); + } + + const originalFolderPath = pathParts.join('/'); + const updatedFolderPath = `workspace-${workspaceId}/${originalFolderPath}`; + + try { + await this.fileStorageService.move({ + from: { folderPath: originalFolderPath, filename }, + to: { folderPath: updatedFolderPath, filename }, + }); + } catch (error) { + if ( + error instanceof FileStorageException && + error.code === FileStorageExceptionCode.FILE_NOT_FOUND + ) { + this.logger.error(`File not found: ${fullPath}`); + } else { + this.logger.error(`Error moving file ${fullPath}: ${error}`); + } + + return; + } + + return { id, updatedFolderPath }; + }; + + const movePromises = filesToMove.map((file) => limit(() => moveFile(file))); + + const results = await Promise.all(movePromises); + + return results.filter( + (result): result is { id: string; updatedFolderPath: string } => + Boolean(result), + ); + } +} diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.command.ts index 696cbac3e..e089eb673 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.command.ts @@ -5,8 +5,9 @@ import { MigrateLinkFieldsToLinksCommand } from 'src/database/commands/upgrade-v import { MigrateMessageChannelSyncStatusEnumCommand } from 'src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command'; import { SetWorkspaceActivationStatusCommand } from 'src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command'; import { UpdateActivitiesCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-activities.command'; +import { UpdateFileFolderStructureCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command'; -interface Options { +interface UpdateTo0_23CommandOptions { workspaceId?: string; } @@ -16,6 +17,7 @@ interface Options { }) export class UpgradeTo0_23Command extends CommandRunner { constructor( + private readonly updateFileFolderStructureCommandOptions: UpdateFileFolderStructureCommand, private readonly migrateLinkFieldsToLinks: MigrateLinkFieldsToLinksCommand, private readonly migrateDomainNameFromTextToLinks: MigrateDomainNameFromTextToLinksCommand, private readonly migrateMessageChannelSyncStatusEnumCommand: MigrateMessageChannelSyncStatusEnumCommand, @@ -35,7 +37,10 @@ export class UpgradeTo0_23Command extends CommandRunner { return value; } - async run(_passedParam: string[], options: Options): Promise { + async run( + _passedParam: string[], + options: UpdateTo0_23CommandOptions, + ): Promise { await this.migrateLinkFieldsToLinks.run(_passedParam, options); await this.migrateDomainNameFromTextToLinks.run(_passedParam, options); await this.migrateMessageChannelSyncStatusEnumCommand.run( @@ -43,6 +48,10 @@ export class UpgradeTo0_23Command extends CommandRunner { options, ); await this.setWorkspaceActivationStatusCommand.run(_passedParam, options); + await this.updateFileFolderStructureCommandOptions.run( + _passedParam, + options, + ); await this.updateActivitiesCommand.run(_passedParam, options); } } diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts index c259f63cf..c7121e27d 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts @@ -6,10 +6,12 @@ import { MigrateLinkFieldsToLinksCommand } from 'src/database/commands/upgrade-v import { MigrateMessageChannelSyncStatusEnumCommand } from 'src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command'; import { SetWorkspaceActivationStatusCommand } from 'src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command'; import { UpdateActivitiesCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-activities.command'; +import { UpdateFileFolderStructureCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command'; import { UpgradeTo0_23Command } from 'src/database/commands/upgrade-version/0-23/0-23-upgrade-version.command'; import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { BillingModule } from 'src/engine/core-modules/billing/billing.module'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { FileStorageModule } from 'src/engine/integrations/file-storage/file-storage.module'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; @@ -22,6 +24,9 @@ import { ViewModule } from 'src/modules/view/view.module'; @Module({ imports: [ TypeOrmModule.forFeature([Workspace], 'core'), + FileStorageModule, + TypeORMModule, + DataSourceModule, WorkspaceCacheVersionModule, FieldMetadataModule, DataSourceModule, @@ -34,6 +39,8 @@ import { ViewModule } from 'src/modules/view/view.module'; ObjectMetadataModule, ], providers: [ + UpdateFileFolderStructureCommand, + UpgradeTo0_23Command, MigrateLinkFieldsToLinksCommand, MigrateDomainNameFromTextToLinksCommand, MigrateMessageChannelSyncStatusEnumCommand, diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/index.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/index.ts index d1e72fb16..a82d328d6 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/index.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/index.ts @@ -1,6 +1,7 @@ -import { QueryResultGettersFactory } from './query-result-getters.factory'; -import { RecordPositionFactory } from './record-position.factory'; import { QueryRunnerArgsFactory } from './query-runner-args.factory'; +import { RecordPositionFactory } from './record-position.factory'; + +import { QueryResultGettersFactory } from './query-result-getters/query-result-getters.factory'; export const workspaceQueryRunnerFactories = [ QueryRunnerArgsFactory, diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters.factory.ts deleted file mode 100644 index 5c61aeb09..000000000 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters.factory.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { addMilliseconds } from 'date-fns'; -import ms from 'ms'; - -import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface'; - -import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; - -@Injectable() -export class QueryResultGettersFactory { - constructor( - private readonly tokenService: TokenService, - private readonly environmentService: EnvironmentService, - ) {} - - async create( - result: Result, - objectMetadataItem: ObjectMetadataInterface, - ): Promise { - // TODO: look for file type once implemented - switch (objectMetadataItem.nameSingular) { - case 'attachment': - return this.applyAttachmentGetters(result); - default: - return result; - } - } - - private async applyAttachmentGetters( - attachments: any, - ): Promise { - if (!attachments || !attachments.edges) { - return attachments; - } - - const fileTokenExpiresIn = this.environmentService.get( - 'FILE_TOKEN_EXPIRES_IN', - ); - const secret = this.environmentService.get('FILE_TOKEN_SECRET'); - - const mappedEdges = await Promise.all( - attachments.edges.map(async (attachment: any) => { - if (!attachment.node.id || !attachment?.node?.fullPath) { - return attachment; - } - - const expirationDate = addMilliseconds( - new Date(), - ms(fileTokenExpiresIn), - ); - - const signedPayload = await this.tokenService.encodePayload( - { - expiration_date: expirationDate, - attachment_id: attachment.node.id, - }, - { - secret, - }, - ); - - attachment.node.fullPath = `${attachment.node.fullPath}?token=${signedPayload}`; - - return attachment; - }), - ); - - return { - ...attachments, - edges: mappedEdges, - } as Result; - } -} diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/attachment-query-result-getter.handler.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/attachment-query-result-getter.handler.ts new file mode 100644 index 000000000..f2c00bbb0 --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/attachment-query-result-getter.handler.ts @@ -0,0 +1,45 @@ +import { addMilliseconds } from 'date-fns'; +import ms from 'ms'; + +import { QueryResultGuetterHandlerInterface } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface'; + +import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; +import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; + +export class AttachmentQueryResultGetterHandler + implements QueryResultGuetterHandlerInterface +{ + constructor( + private readonly tokenService: TokenService, + private readonly environmentService: EnvironmentService, + ) {} + + async process(attachment: any, workspaceId: string): Promise { + if (!attachment.id || !attachment?.fullPath) { + return attachment; + } + + const fileTokenExpiresIn = this.environmentService.get( + 'FILE_TOKEN_EXPIRES_IN', + ); + const secret = this.environmentService.get('FILE_TOKEN_SECRET'); + + const expirationDate = addMilliseconds(new Date(), ms(fileTokenExpiresIn)); + + const signedPayload = await this.tokenService.encodePayload( + { + expiration_date: expirationDate, + attachment_id: attachment.id, + workspace_id: workspaceId, + }, + { + secret, + }, + ); + + return { + ...attachment, + fullPath: `${attachment.fullPath}?token=${signedPayload}`, + }; + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface.ts new file mode 100644 index 000000000..742a3fe32 --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface.ts @@ -0,0 +1,3 @@ +export interface QueryResultGuetterHandlerInterface { + process(result: any, workspaceId: string): Promise; +} diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory.ts new file mode 100644 index 000000000..eaed388df --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory.ts @@ -0,0 +1,73 @@ +import { Injectable } from '@nestjs/common'; + +import { QueryResultGuetterHandlerInterface } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface'; +import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface'; + +import { AttachmentQueryResultGetterHandler } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/attachment-query-result-getter.handler'; +import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; +import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; + +@Injectable() +export class QueryResultGettersFactory { + private handlers: Map; + + constructor( + private readonly tokenService: TokenService, + private readonly environmentService: EnvironmentService, + ) { + this.initializeHandlers(); + } + + private initializeHandlers() { + this.handlers = new Map([ + [ + 'attachment', + new AttachmentQueryResultGetterHandler( + this.tokenService, + this.environmentService, + ), + ], + ]); + } + + async create( + result: any, + objectMetadataItem: ObjectMetadataInterface, + workspaceId: string, + ): Promise { + const handler = this.getHandler(objectMetadataItem.nameSingular); + + if (result.edges) { + return { + ...result, + edges: await Promise.all( + result.edges.map(async (edge: any) => ({ + ...edge, + node: await handler.process(edge.node, workspaceId), + })), + ), + }; + } + + if (result.records) { + return { + ...result, + records: await Promise.all( + result.records.map( + async (item: any) => await handler.process(item, workspaceId), + ), + ), + }; + } + + return await handler.process(result, workspaceId); + } + + private getHandler(objectType: string): QueryResultGuetterHandlerInterface { + return ( + this.handlers.get(objectType) || { + process: (result: any) => result, + } + ); + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts index 6bb6156fd..1a5de2956 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/workspace-query-runner.service.ts @@ -25,7 +25,7 @@ import { import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface'; import { WorkspaceQueryBuilderFactory } from 'src/engine/api/graphql/workspace-query-builder/workspace-query-builder.factory'; -import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters.factory'; +import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory'; import { QueryRunnerArgsFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory'; import { CallWebhookJobsJob, @@ -125,6 +125,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, '', + workspaceId, ); } @@ -167,6 +168,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, '', + workspaceId, ); return parsedResult?.edges?.[0]?.node; @@ -235,6 +237,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, '', + workspaceId, true, ); } @@ -283,6 +286,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, 'insertInto', + workspaceId, ) )?.records; @@ -418,6 +422,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, 'update', + workspaceId, ) )?.records; @@ -485,6 +490,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, 'update', + workspaceId, ) )?.records; @@ -555,6 +561,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, 'deleteFrom', + workspaceId, ) )?.records; @@ -618,6 +625,7 @@ export class WorkspaceQueryRunnerService { result, objectMetadataItem, 'deleteFrom', + workspaceId, ) )?.records; @@ -721,6 +729,7 @@ export class WorkspaceQueryRunnerService { graphqlResult: PGGraphQLResult | undefined, objectMetadataItem: ObjectMetadataInterface, command: string, + workspaceId: string, isMultiQuery = false, ): Promise { const entityKey = `${command}${computeObjectTargetTable( @@ -767,6 +776,7 @@ export class WorkspaceQueryRunnerService { const resultWithGetters = await this.queryResultGettersFactory.create( result, objectMetadataItem, + workspaceId, ); return parseResult(resultWithGetters); @@ -780,7 +790,7 @@ export class WorkspaceQueryRunnerService { ): Promise { const result = await this.execute(query, workspaceId); - return this.parseResult(result, objectMetadataItem, command); + return this.parseResult(result, objectMetadataItem, command, workspaceId); } async triggerWebhooks( diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/sign-in-up.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/sign-in-up.service.ts index 90bfd8f6e..b53284dd2 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/sign-in-up.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/sign-in-up.service.ts @@ -75,11 +75,6 @@ export class SignInUpService { const passwordHash = password ? await hashPassword(password) : undefined; - let imagePath: string | undefined; - - if (picture) { - imagePath = await this.uploadPicture(picture); - } const existingUser = await this.userRepository.findOne({ where: { email: email, @@ -103,7 +98,7 @@ export class SignInUpService { workspaceInviteHash, firstName, lastName, - imagePath, + picture, existingUser, }); } @@ -113,7 +108,7 @@ export class SignInUpService { passwordHash, firstName, lastName, - imagePath, + picture, }); } @@ -126,7 +121,7 @@ export class SignInUpService { workspaceInviteHash, firstName, lastName, - imagePath, + picture, existingUser, }: { email: string; @@ -134,7 +129,7 @@ export class SignInUpService { workspaceInviteHash: string; firstName: string; lastName: string; - imagePath: string | undefined; + picture: SignInUpServiceInput['picture']; existingUser: User | null; }) { const workspace = await this.workspaceRepository.findOneBy({ @@ -162,6 +157,8 @@ export class SignInUpService { return Object.assign(existingUser, updatedUser); } + const imagePath = await this.uploadPicture(picture, workspace.id); + const userToCreate = this.userRepository.create({ email: email, firstName: firstName, @@ -185,13 +182,13 @@ export class SignInUpService { passwordHash, firstName, lastName, - imagePath, + picture, }: { email: string; passwordHash: string | undefined; firstName: string; lastName: string; - imagePath: string | undefined; + picture: SignInUpServiceInput['picture']; }) { assert( !this.environmentService.get('IS_SIGN_UP_DISABLED'), @@ -208,6 +205,8 @@ export class SignInUpService { const workspace = await this.workspaceRepository.save(workspaceToCreate); + const imagePath = await this.uploadPicture(picture, workspace.id); + const userToCreate = this.userRepository.create({ email: email, firstName: firstName, @@ -225,7 +224,14 @@ export class SignInUpService { return user; } - async uploadPicture(picture: string): Promise { + async uploadPicture( + picture: string | null | undefined, + workspaceId: string, + ): Promise { + if (!picture) { + return; + } + const buffer = await getImageBufferFromUrl( picture, this.httpService.axiosRef, @@ -238,6 +244,7 @@ export class SignInUpService { filename: `${v4()}.${type?.ext}`, mimeType: type?.mime, fileFolder: FileFolder.ProfilePicture, + workspaceId, }); return paths[0]; diff --git a/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts b/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts index 39f2b8c70..a564460fc 100644 --- a/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts @@ -1,12 +1,17 @@ -import { Controller, Get, Param, Res, UseGuards } from '@nestjs/common'; +import { Controller, Get, Param, Req, Res, UseGuards } from '@nestjs/common'; import { Response } from 'express'; -import { FilePathGuard } from 'src/engine/core-modules/file/guards/file-path-guard'; +import { + FileStorageException, + FileStorageExceptionCode, +} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception'; + import { checkFilePath, checkFilename, } from 'src/engine/core-modules/file/file.utils'; +import { FilePathGuard } from 'src/engine/core-modules/file/guards/file-path-guard'; import { FileService } from 'src/engine/core-modules/file/services/file.service'; // TODO: Add cookie authentication @@ -15,23 +20,38 @@ import { FileService } from 'src/engine/core-modules/file/services/file.service' export class FileController { constructor(private readonly fileService: FileService) {} - /** - * Serve files from local storage - * We recommend using an s3 bucket for production - */ @Get('*/:filename') - async getFile(@Param() params: string[], @Res() res: Response) { + async getFile( + @Param() params: string[], + @Res() res: Response, + @Req() req: Request, + ) { const folderPath = checkFilePath(params[0]); const filename = checkFilename(params['filename']); - const fileStream = await this.fileService.getFileStream( - folderPath, - filename, - ); - fileStream.on('error', () => { - res.status(404).send({ error: 'File not found' }); - }); + const workspaceId = (req as any)?.workspaceId; - fileStream.pipe(res); + if (!workspaceId) { + return res.status(401).send({ error: 'Unauthorized' }); + } + + try { + const fileStream = await this.fileService.getFileStream( + folderPath, + filename, + workspaceId, + ); + + fileStream.pipe(res); + } catch (error) { + if ( + error instanceof FileStorageException && + error.code === FileStorageExceptionCode.FILE_NOT_FOUND + ) { + return res.status(404).send({ error: 'File not found' }); + } + + return res.status(500).send({ error: 'Internal server error' }); + } } } diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver.ts index 1541bb63e..8e5bae940 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver.ts @@ -1,14 +1,16 @@ -import { Args, Mutation, Resolver } from '@nestjs/graphql'; import { UseGuards } from '@nestjs/common'; +import { Args, Mutation, Resolver } from '@nestjs/graphql'; -import { GraphQLUpload, FileUpload } from 'graphql-upload'; +import { FileUpload, GraphQLUpload } from 'graphql-upload'; import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface'; import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; +import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; -import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; @UseGuards(JwtAuthGuard, DemoEnvGuard) @Resolver() @@ -17,6 +19,7 @@ export class FileUploadResolver { @Mutation(() => String) async uploadFile( + @AuthWorkspace() { id: workspaceId }: Workspace, @Args({ name: 'file', type: () => GraphQLUpload }) { createReadStream, filename, mimetype }: FileUpload, @Args('fileFolder', { type: () => FileFolder, nullable: true }) @@ -30,6 +33,7 @@ export class FileUploadResolver { filename, mimeType: mimetype, fileFolder, + workspaceId, }); return path; @@ -37,6 +41,7 @@ export class FileUploadResolver { @Mutation(() => String) async uploadImage( + @AuthWorkspace() { id: workspaceId }: Workspace, @Args({ name: 'file', type: () => GraphQLUpload }) { createReadStream, filename, mimetype }: FileUpload, @Args('fileFolder', { type: () => FileFolder, nullable: true }) @@ -50,6 +55,7 @@ export class FileUploadResolver { filename, mimeType: mimetype, fileFolder, + workspaceId, }); return paths[0]; diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts index 7405aeb56..a50f5a012 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts @@ -1,15 +1,15 @@ import { Injectable } from '@nestjs/common'; +import DOMPurify from 'dompurify'; +import { JSDOM } from 'jsdom'; import sharp from 'sharp'; import { v4 as uuidV4 } from 'uuid'; -import { JSDOM } from 'jsdom'; -import DOMPurify from 'dompurify'; import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface'; -import { getCropSize } from 'src/utils/image'; import { settings } from 'src/engine/constants/settings'; import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service'; +import { getCropSize } from 'src/utils/image'; @Injectable() export class FileUploadService { @@ -19,18 +19,18 @@ export class FileUploadService { file, filename, mimeType, - fileFolder, + folder, }: { file: Buffer | Uint8Array | string; filename: string; mimeType: string | undefined; - fileFolder: FileFolder; + folder: string; }) { await this.fileStorage.write({ file, name: filename, mimeType, - folder: fileFolder, + folder, }); } @@ -58,21 +58,24 @@ export class FileUploadService { filename, mimeType, fileFolder, + workspaceId, }: { file: Buffer | Uint8Array | string; filename: string; mimeType: string | undefined; fileFolder: FileFolder; + workspaceId: string; }) { const ext = filename.split('.')?.[1]; const id = uuidV4(); const name = `${id}${ext ? `.${ext}` : ''}`; + const folder = this.getWorkspaceFolderName(workspaceId, fileFolder); await this._uploadFile({ file: this._sanitizeFile({ file, ext, mimeType }), filename: name, mimeType, - fileFolder, + folder, }); return { @@ -87,11 +90,13 @@ export class FileUploadService { filename, mimeType, fileFolder, + workspaceId, }: { file: Buffer | Uint8Array | string; filename: string; mimeType: string | undefined; fileFolder: FileFolder; + workspaceId: string; }) { const ext = filename.split('.')?.[1]; const id = uuidV4(); @@ -117,6 +122,7 @@ export class FileUploadService { await Promise.all( images.map(async (image, index) => { const buffer = await image.toBuffer(); + const folder = this.getWorkspaceFolderName(workspaceId, fileFolder); paths.push(`${fileFolder}/${cropSizes[index]}/${name}`); @@ -124,7 +130,7 @@ export class FileUploadService { file: buffer, filename: `${cropSizes[index]}/${name}`, mimeType, - fileFolder, + folder, }); }), ); @@ -135,4 +141,8 @@ export class FileUploadService { paths, }; } + + private getWorkspaceFolderName(workspaceId: string, fileFolder: FileFolder) { + return `workspace-${workspaceId}/${fileFolder}`; + } } diff --git a/packages/twenty-server/src/engine/core-modules/file/file.utils.spec.ts b/packages/twenty-server/src/engine/core-modules/file/file.utils.spec.ts new file mode 100644 index 000000000..d6a19d3f9 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/file/file.utils.spec.ts @@ -0,0 +1,61 @@ +import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface'; + +import { + checkFilename, + checkFilePath, +} from 'src/engine/core-modules/file/file.utils'; + +describe('FileUtils', () => { + describe('checkFilePath', () => { + it('should return sanitized file path', () => { + const filePath = `${FileFolder.Attachment}\0`; + const sanitizedFilePath = checkFilePath(filePath); + + expect(sanitizedFilePath).toBe(`${FileFolder.Attachment}`); + }); + + it('should return sanitized file path with size', () => { + const filePath = `${FileFolder.ProfilePicture}\0/original`; + const sanitizedFilePath = checkFilePath(filePath); + + expect(sanitizedFilePath).toBe(`${FileFolder.ProfilePicture}/original`); + }); + + it('should throw an error for invalid image size', () => { + const filePath = `${FileFolder.ProfilePicture}\0/invalid-size`; + + expect(() => checkFilePath(filePath)).toThrow( + `Size invalid-size is not allowed`, + ); + }); + + it('should throw an error for invalid folder', () => { + const filePath = `invalid-folder`; + + expect(() => checkFilePath(filePath)).toThrow( + `Folder invalid-folder is not allowed`, + ); + }); + }); + + describe('checkFilename', () => { + it('should return sanitized filename', () => { + const filename = `${FileFolder.Attachment}\0.png`; + const sanitizedFilename = checkFilename(filename); + + expect(sanitizedFilename).toBe(`${FileFolder.Attachment}.png`); + }); + + it('should throw an error for invalid filename', () => { + const filename = `invalid-filename`; + + expect(() => checkFilename(filename)).toThrow(`Filename is not allowed`); + }); + + it('should throw an error for invalid filename', () => { + const filename = `\0`; + + expect(() => checkFilename(filename)).toThrow(`Filename is not allowed`); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/file/file.utils.ts b/packages/twenty-server/src/engine/core-modules/file/file.utils.ts index ef05b9419..0d7b042d2 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file.utils.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file.utils.ts @@ -4,8 +4,8 @@ import { basename } from 'path'; import { KebabCase } from 'type-fest'; -import { kebabCase } from 'src/utils/kebab-case'; import { settings } from 'src/engine/constants/settings'; +import { kebabCase } from 'src/utils/kebab-case'; import { FileFolder } from './interfaces/file-folder.interface'; diff --git a/packages/twenty-server/src/engine/core-modules/file/guards/file-path-guard.ts b/packages/twenty-server/src/engine/core-modules/file/guards/file-path-guard.ts index 988916473..082d6383c 100644 --- a/packages/twenty-server/src/engine/core-modules/file/guards/file-path-guard.ts +++ b/packages/twenty-server/src/engine/core-modules/file/guards/file-path-guard.ts @@ -1,13 +1,13 @@ import { - Injectable, CanActivate, ExecutionContext, HttpException, HttpStatus, + Injectable, } from '@nestjs/common'; -import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; +import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; @Injectable() export class FilePathGuard implements CanActivate { @@ -17,25 +17,34 @@ export class FilePathGuard implements CanActivate { ) {} async canActivate(context: ExecutionContext): Promise { - const query = context.switchToHttp().getRequest().query; + const request = context.switchToHttp().getRequest(); + const query = request.query; if (query && query['token']) { - return !(await this.isExpired(query['token'])); + const payloadToDecode = query['token']; + const decodedPayload = await this.tokenService.decodePayload( + payloadToDecode, + { + secret: this.environmentService.get('FILE_TOKEN_SECRET'), + }, + ); + + const expirationDate = decodedPayload?.['expiration_date']; + const workspaceId = decodedPayload?.['workspace_id']; + + const isExpired = await this.isExpired(expirationDate); + + if (isExpired) { + return false; + } + + request.workspaceId = workspaceId; } return true; } - private async isExpired(signedExpirationDate: string): Promise { - const decodedPayload = await this.tokenService.decodePayload( - signedExpirationDate, - { - secret: this.environmentService.get('FILE_TOKEN_SECRET'), - }, - ); - - const expirationDate = decodedPayload?.['expiration_date']; - + private async isExpired(expirationDate: string): Promise { if (!expirationDate) { return true; } diff --git a/packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.spec.ts b/packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.spec.ts deleted file mode 100644 index 5a62ebdc4..000000000 --- a/packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.spec.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; - -import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service'; - -import { FileUploadResolver } from './file-upload.resolver'; - -describe('FileUploadResolver', () => { - let resolver: FileUploadResolver; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - FileUploadResolver, - { - provide: FileUploadService, - useValue: {}, - }, - ], - }).compile(); - - resolver = module.get(FileUploadResolver); - }); - - it('should be defined', () => { - expect(resolver).toBeDefined(); - }); -}); diff --git a/packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.ts b/packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.ts deleted file mode 100644 index 1541bb63e..000000000 --- a/packages/twenty-server/src/engine/core-modules/file/resolvers/file-upload.resolver.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Args, Mutation, Resolver } from '@nestjs/graphql'; -import { UseGuards } from '@nestjs/common'; - -import { GraphQLUpload, FileUpload } from 'graphql-upload'; - -import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface'; - -import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service'; -import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; -import { streamToBuffer } from 'src/utils/stream-to-buffer'; -import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; - -@UseGuards(JwtAuthGuard, DemoEnvGuard) -@Resolver() -export class FileUploadResolver { - constructor(private readonly fileUploadService: FileUploadService) {} - - @Mutation(() => String) - async uploadFile( - @Args({ name: 'file', type: () => GraphQLUpload }) - { createReadStream, filename, mimetype }: FileUpload, - @Args('fileFolder', { type: () => FileFolder, nullable: true }) - fileFolder: FileFolder, - ): Promise { - const stream = createReadStream(); - const buffer = await streamToBuffer(stream); - - const { path } = await this.fileUploadService.uploadFile({ - file: buffer, - filename, - mimeType: mimetype, - fileFolder, - }); - - return path; - } - - @Mutation(() => String) - async uploadImage( - @Args({ name: 'file', type: () => GraphQLUpload }) - { createReadStream, filename, mimetype }: FileUpload, - @Args('fileFolder', { type: () => FileFolder, nullable: true }) - fileFolder: FileFolder, - ): Promise { - const stream = createReadStream(); - const buffer = await streamToBuffer(stream); - - const { paths } = await this.fileUploadService.uploadImage({ - file: buffer, - filename, - mimeType: mimetype, - fileFolder, - }); - - return paths[0]; - } -} diff --git a/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts b/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts index 290a13f8b..d4b1b021d 100644 --- a/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts +++ b/packages/twenty-server/src/engine/core-modules/file/services/file.service.ts @@ -1,15 +1,42 @@ import { Injectable } from '@nestjs/common'; +import { Stream } from 'stream'; + +import { + FileStorageException, + FileStorageExceptionCode, +} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception'; + import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service'; @Injectable() export class FileService { constructor(private readonly fileStorageService: FileStorageService) {} - async getFileStream(folderPath: string, filename: string) { - return this.fileStorageService.read({ - folderPath, - filename, - }); + async getFileStream( + folderPath: string, + filename: string, + workspaceId: string, + ): Promise { + const workspaceFolderPath = `workspace-${workspaceId}/${folderPath}`; + + try { + return await this.fileStorageService.read({ + folderPath: workspaceFolderPath, + filename, + }); + } catch (error) { + // TODO: Remove this fallback when all files are moved to workspace folders + if ( + error instanceof FileStorageException && + error.code === FileStorageExceptionCode.FILE_NOT_FOUND + ) { + return await this.fileStorageService.read({ + folderPath, + filename, + }); + } + throw error; + } } } diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts index 3dce22b89..12627dc64 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts @@ -9,6 +9,7 @@ import { } from '@nestjs/graphql'; import { InjectRepository } from '@nestjs/typeorm'; +import assert from 'assert'; import crypto from 'crypto'; import { GraphQLJSONObject } from 'graphql-type-json'; @@ -32,7 +33,6 @@ import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; -import { assert } from 'src/utils/assert'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; const getHMACKey = (email?: string, key?: string | null) => { @@ -117,6 +117,7 @@ export class UserResolver { @Mutation(() => String) async uploadProfilePicture( @AuthUser() { id }: User, + @AuthWorkspace() { id: workspaceId }: Workspace, @Args({ name: 'file', type: () => GraphQLUpload }) { createReadStream, filename, mimetype }: FileUpload, ): Promise { @@ -133,6 +134,7 @@ export class UserResolver { filename, mimeType: mimetype, fileFolder, + workspaceId, }); return paths[0]; diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts index 43542a22b..dff5ed2ee 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts @@ -85,6 +85,7 @@ export class WorkspaceResolver { filename, mimeType: mimetype, fileFolder, + workspaceId: id, }); await this.workspaceService.updateOne(id, { diff --git a/packages/twenty-server/src/engine/integrations/file-storage/drivers/interfaces/storage-driver.interface.ts b/packages/twenty-server/src/engine/integrations/file-storage/drivers/interfaces/storage-driver.interface.ts index fb46892e7..08c9f0708 100644 --- a/packages/twenty-server/src/engine/integrations/file-storage/drivers/interfaces/storage-driver.interface.ts +++ b/packages/twenty-server/src/engine/integrations/file-storage/drivers/interfaces/storage-driver.interface.ts @@ -9,4 +9,8 @@ export interface StorageDriver { folder: string; mimeType: string | undefined; }): Promise; + move(params: { + from: { folderPath: string; filename: string }; + to: { folderPath: string; filename: string }; + }): Promise; } diff --git a/packages/twenty-server/src/engine/integrations/file-storage/drivers/local.driver.ts b/packages/twenty-server/src/engine/integrations/file-storage/drivers/local.driver.ts index e61a9338a..3a2e46f93 100644 --- a/packages/twenty-server/src/engine/integrations/file-storage/drivers/local.driver.ts +++ b/packages/twenty-server/src/engine/integrations/file-storage/drivers/local.driver.ts @@ -1,8 +1,13 @@ -import * as fs from 'fs/promises'; import { createReadStream, existsSync } from 'fs'; -import { join, dirname } from 'path'; +import * as fs from 'fs/promises'; +import { dirname, join } from 'path'; import { Readable } from 'stream'; +import { + FileStorageException, + FileStorageExceptionCode, +} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception'; + import { StorageDriver } from './interfaces/storage-driver.interface'; export interface LocalDriverOptions { @@ -65,6 +70,49 @@ export class LocalDriver implements StorageDriver { params.filename, ); - return createReadStream(filePath); + try { + return createReadStream(filePath); + } catch (error) { + if (error.code === 'ENOENT') { + throw new FileStorageException( + 'File not found', + FileStorageExceptionCode.FILE_NOT_FOUND, + ); + } + + throw error; + } + } + + async move(params: { + from: { folderPath: string; filename: string }; + to: { folderPath: string; filename: string }; + }): Promise { + const fromPath = join( + `${this.options.storagePath}/`, + params.from.folderPath, + params.from.filename, + ); + + const toPath = join( + `${this.options.storagePath}/`, + params.to.folderPath, + params.to.filename, + ); + + await this.createFolder(dirname(toPath)); + + try { + await fs.rename(fromPath, toPath); + } catch (error) { + if (error.code === 'ENOENT') { + throw new FileStorageException( + 'File not found', + FileStorageExceptionCode.FILE_NOT_FOUND, + ); + } + + throw error; + } } } diff --git a/packages/twenty-server/src/engine/integrations/file-storage/drivers/s3.driver.ts b/packages/twenty-server/src/engine/integrations/file-storage/drivers/s3.driver.ts index 9d0627071..7ced7608d 100644 --- a/packages/twenty-server/src/engine/integrations/file-storage/drivers/s3.driver.ts +++ b/packages/twenty-server/src/engine/integrations/file-storage/drivers/s3.driver.ts @@ -1,11 +1,13 @@ import { Readable } from 'stream'; import { + CopyObjectCommand, CreateBucketCommandInput, DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadBucketCommandInput, + HeadObjectCommand, ListObjectsV2Command, NotFound, PutObjectCommand, @@ -13,6 +15,11 @@ import { S3ClientConfig, } from '@aws-sdk/client-s3'; +import { + FileStorageException, + FileStorageExceptionCode, +} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception'; + import { StorageDriver } from './interfaces/storage-driver.interface'; export interface S3DriverOptions extends S3ClientConfig { @@ -115,13 +122,69 @@ export class S3Driver implements StorageDriver { Key: `${params.folderPath}/${params.filename}`, Bucket: this.bucketName, }); - const file = await this.s3Client.send(command); - if (!file || !file.Body || !(file.Body instanceof Readable)) { - throw new Error('Unable to get file stream'); + try { + const file = await this.s3Client.send(command); + + if (!file || !file.Body || !(file.Body instanceof Readable)) { + throw new Error('Unable to get file stream'); + } + + return Readable.from(file.Body); + } catch (error) { + if (error.name === 'NoSuchKey') { + throw new FileStorageException( + 'File not found', + FileStorageExceptionCode.FILE_NOT_FOUND, + ); + } + + throw error; } + } - return Readable.from(file.Body); + async move(params: { + from: { folderPath: string; filename: string }; + to: { folderPath: string; filename: string }; + }): Promise { + const fromKey = `${params.from.folderPath}/${params.from.filename}`; + const toKey = `${params.to.folderPath}/${params.to.filename}`; + + try { + // Check if the source file exists + await this.s3Client.send( + new HeadObjectCommand({ + Bucket: this.bucketName, + Key: fromKey, + }), + ); + + // Copy the object to the new location + await this.s3Client.send( + new CopyObjectCommand({ + CopySource: `${this.bucketName}/${fromKey}`, + Bucket: this.bucketName, + Key: toKey, + }), + ); + + // Delete the original object + await this.s3Client.send( + new DeleteObjectCommand({ + Bucket: this.bucketName, + Key: fromKey, + }), + ); + } catch (error) { + if (error.name === 'NotFound') { + throw new FileStorageException( + 'File not found', + FileStorageExceptionCode.FILE_NOT_FOUND, + ); + } + // For other errors, throw the original error + throw error; + } } async checkBucketExists(args: HeadBucketCommandInput) { diff --git a/packages/twenty-server/src/engine/integrations/file-storage/file-storage.service.ts b/packages/twenty-server/src/engine/integrations/file-storage/file-storage.service.ts index 063170614..65133f991 100644 --- a/packages/twenty-server/src/engine/integrations/file-storage/file-storage.service.ts +++ b/packages/twenty-server/src/engine/integrations/file-storage/file-storage.service.ts @@ -26,4 +26,11 @@ export class FileStorageService implements StorageDriver { read(params: { folderPath: string; filename: string }): Promise { return this.driver.read(params); } + + move(params: { + from: { folderPath: string; filename: string }; + to: { folderPath: string; filename: string }; + }): Promise { + return this.driver.move(params); + } } diff --git a/packages/twenty-server/src/engine/integrations/file-storage/interfaces/file-storage-exception.ts b/packages/twenty-server/src/engine/integrations/file-storage/interfaces/file-storage-exception.ts new file mode 100644 index 000000000..82f065ff1 --- /dev/null +++ b/packages/twenty-server/src/engine/integrations/file-storage/interfaces/file-storage-exception.ts @@ -0,0 +1,12 @@ +import { CustomException } from 'src/utils/custom-exception'; + +export class FileStorageException extends CustomException { + code: FileStorageExceptionCode; + constructor(message: string, code: FileStorageExceptionCode) { + super(message, code); + } +} + +export enum FileStorageExceptionCode { + FILE_NOT_FOUND = 'FILE_NOT_FOUND', +}