file storage workspace id prefix (#6230)

closes https://github.com/twentyhq/twenty/issues/6155

just an idea, i guess this could work well, but im open for discussion

---------

Co-authored-by: Weiko <corentin@twenty.com>
This commit is contained in:
rostaklein
2024-08-01 18:07:22 +02:00
committed by GitHub
parent 5c92ab937e
commit a424c63476
26 changed files with 727 additions and 231 deletions

View File

@ -0,0 +1,230 @@
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import chalk from 'chalk';
import { Command, CommandRunner, Option } from 'nest-commander';
import pLimit from 'p-limit';
import { Like, Repository } from 'typeorm';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
interface UpdateFileFolderStructureCommandOptions {
workspaceId?: string;
}
@Command({
name: 'upgrade-0-23:update-file-folder-structure',
description: 'Update file folder structure (prefixed per workspace)',
})
export class UpdateFileFolderStructureCommand extends CommandRunner {
private readonly logger = new Logger(UpdateFileFolderStructureCommand.name);
constructor(
@InjectRepository(Workspace, 'core')
private readonly workspaceRepository: Repository<Workspace>,
private readonly typeORMService: TypeORMService,
private readonly dataSourceService: DataSourceService,
private readonly fileStorageService: FileStorageService,
) {
super();
}
@Option({
flags: '-w, --workspace-id [workspace_id]',
description: 'workspace id. Command runs on all workspaces if not provided',
required: false,
})
parseWorkspaceId(value: string): string {
return value;
}
async run(
_passedParam: string[],
options: UpdateFileFolderStructureCommandOptions,
): Promise<void> {
const workspaceIds = options.workspaceId
? [options.workspaceId]
: (await this.workspaceRepository.find()).map(
(workspace) => workspace.id,
);
if (!workspaceIds.length) {
this.logger.log(chalk.yellow('No workspace found'));
return;
}
this.logger.log(
chalk.green(`Running command on ${workspaceIds.length} workspaces`),
);
for (const workspaceId of workspaceIds) {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
workspaceId,
);
if (!dataSourceMetadata) {
this.logger.log(
`Could not find dataSourceMetadata for workspace ${workspaceId}`,
);
continue;
}
const workspaceDataSource =
await this.typeORMService.connectToDataSource(dataSourceMetadata);
if (!workspaceDataSource) {
throw new Error(
`Could not connect to dataSource for workspace ${workspaceId}`,
);
}
const workspaceQueryRunner = workspaceDataSource.createQueryRunner();
const attachmentsToMove = (await workspaceQueryRunner.query(
`SELECT id, "fullPath" FROM "${dataSourceMetadata.schema}"."attachment" WHERE "fullPath" LIKE '${FileFolder.Attachment}/%'`,
)) as { id: string; fullPath: string }[];
const workspaceMemberAvatarsToMove = (await workspaceQueryRunner.query(
`SELECT id, "avatarUrl" as "fullPath" FROM "${dataSourceMetadata.schema}"."workspaceMember" WHERE "avatarUrl" LIKE '${FileFolder.ProfilePicture}/%'`,
)) as { id: string; fullPath: string }[];
const personAvatarsToMove = (await workspaceQueryRunner.query(
`SELECT id, "avatarUrl" as "fullPath" FROM "${dataSourceMetadata.schema}"."person" WHERE "avatarUrl" LIKE '${FileFolder.PersonPicture}/%'`,
)) as { id: string; fullPath: string }[];
const workspacePictureToMove = await this.workspaceRepository.findOneBy({
id: workspaceId,
logo: Like(`${FileFolder.WorkspaceLogo}/%`),
});
try {
const updatedAttachments = await this.moveFiles(
workspaceId,
attachmentsToMove,
);
this.logger.log(
chalk.green(
`Moved ${updatedAttachments.length} attachments in workspace ${workspaceId}`,
),
);
} catch (e) {
this.logger.error(e);
}
try {
const updatedWorkspaceMemberAvatars = await this.moveFiles(
workspaceId,
workspaceMemberAvatarsToMove,
);
this.logger.log(
chalk.green(
`Moved ${updatedWorkspaceMemberAvatars.length} workspaceMemberAvatars in workspace ${workspaceId}`,
),
);
} catch (e) {
this.logger.error(e);
}
try {
const updatedPersonAvatars = await this.moveFiles(
workspaceId,
personAvatarsToMove,
);
this.logger.log(
chalk.green(
`Moved ${updatedPersonAvatars.length} personAvatars in workspace ${workspaceId}`,
),
);
} catch (e) {
this.logger.error(e);
}
if (workspacePictureToMove?.logo) {
await this.moveFiles(workspaceId, [
{
id: workspacePictureToMove.id,
fullPath: workspacePictureToMove.logo,
},
]);
this.logger.log(
chalk.green(`Moved workspacePicture in workspace ${workspaceId}`),
);
}
this.logger.log(
chalk.green(`Running command on workspace ${workspaceId} done`),
);
}
this.logger.log(chalk.green(`Command completed!`));
}
private async moveFiles(
workspaceId: string,
filesToMove: { id: string; fullPath: string }[],
): Promise<Array<{ id: string; updatedFolderPath: string }>> {
const batchSize = 20;
const limit = pLimit(batchSize);
const moveFile = async ({
id,
fullPath,
}: {
id: string;
fullPath: string;
}) => {
const pathParts = fullPath.split('/');
const filename = pathParts.pop();
if (!filename) {
throw new Error(`Filename is empty for file ID: ${id}`);
}
const originalFolderPath = pathParts.join('/');
const updatedFolderPath = `workspace-${workspaceId}/${originalFolderPath}`;
try {
await this.fileStorageService.move({
from: { folderPath: originalFolderPath, filename },
to: { folderPath: updatedFolderPath, filename },
});
} catch (error) {
if (
error instanceof FileStorageException &&
error.code === FileStorageExceptionCode.FILE_NOT_FOUND
) {
this.logger.error(`File not found: ${fullPath}`);
} else {
this.logger.error(`Error moving file ${fullPath}: ${error}`);
}
return;
}
return { id, updatedFolderPath };
};
const movePromises = filesToMove.map((file) => limit(() => moveFile(file)));
const results = await Promise.all(movePromises);
return results.filter(
(result): result is { id: string; updatedFolderPath: string } =>
Boolean(result),
);
}
}

View File

@ -5,8 +5,9 @@ import { MigrateLinkFieldsToLinksCommand } from 'src/database/commands/upgrade-v
import { MigrateMessageChannelSyncStatusEnumCommand } from 'src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command';
import { SetWorkspaceActivationStatusCommand } from 'src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command';
import { UpdateActivitiesCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-activities.command';
import { UpdateFileFolderStructureCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command';
interface Options {
interface UpdateTo0_23CommandOptions {
workspaceId?: string;
}
@ -16,6 +17,7 @@ interface Options {
})
export class UpgradeTo0_23Command extends CommandRunner {
constructor(
private readonly updateFileFolderStructureCommandOptions: UpdateFileFolderStructureCommand,
private readonly migrateLinkFieldsToLinks: MigrateLinkFieldsToLinksCommand,
private readonly migrateDomainNameFromTextToLinks: MigrateDomainNameFromTextToLinksCommand,
private readonly migrateMessageChannelSyncStatusEnumCommand: MigrateMessageChannelSyncStatusEnumCommand,
@ -35,7 +37,10 @@ export class UpgradeTo0_23Command extends CommandRunner {
return value;
}
async run(_passedParam: string[], options: Options): Promise<void> {
async run(
_passedParam: string[],
options: UpdateTo0_23CommandOptions,
): Promise<void> {
await this.migrateLinkFieldsToLinks.run(_passedParam, options);
await this.migrateDomainNameFromTextToLinks.run(_passedParam, options);
await this.migrateMessageChannelSyncStatusEnumCommand.run(
@ -43,6 +48,10 @@ export class UpgradeTo0_23Command extends CommandRunner {
options,
);
await this.setWorkspaceActivationStatusCommand.run(_passedParam, options);
await this.updateFileFolderStructureCommandOptions.run(
_passedParam,
options,
);
await this.updateActivitiesCommand.run(_passedParam, options);
}
}

View File

@ -6,10 +6,12 @@ import { MigrateLinkFieldsToLinksCommand } from 'src/database/commands/upgrade-v
import { MigrateMessageChannelSyncStatusEnumCommand } from 'src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command';
import { SetWorkspaceActivationStatusCommand } from 'src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command';
import { UpdateActivitiesCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-activities.command';
import { UpdateFileFolderStructureCommand } from 'src/database/commands/upgrade-version/0-23/0-23-update-file-folder-structure.command';
import { UpgradeTo0_23Command } from 'src/database/commands/upgrade-version/0-23/0-23-upgrade-version.command';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { FileStorageModule } from 'src/engine/integrations/file-storage/file-storage.module';
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
@ -22,6 +24,9 @@ import { ViewModule } from 'src/modules/view/view.module';
@Module({
imports: [
TypeOrmModule.forFeature([Workspace], 'core'),
FileStorageModule,
TypeORMModule,
DataSourceModule,
WorkspaceCacheVersionModule,
FieldMetadataModule,
DataSourceModule,
@ -34,6 +39,8 @@ import { ViewModule } from 'src/modules/view/view.module';
ObjectMetadataModule,
],
providers: [
UpdateFileFolderStructureCommand,
UpgradeTo0_23Command,
MigrateLinkFieldsToLinksCommand,
MigrateDomainNameFromTextToLinksCommand,
MigrateMessageChannelSyncStatusEnumCommand,

View File

@ -1,6 +1,7 @@
import { QueryResultGettersFactory } from './query-result-getters.factory';
import { RecordPositionFactory } from './record-position.factory';
import { QueryRunnerArgsFactory } from './query-runner-args.factory';
import { RecordPositionFactory } from './record-position.factory';
import { QueryResultGettersFactory } from './query-result-getters/query-result-getters.factory';
export const workspaceQueryRunnerFactories = [
QueryRunnerArgsFactory,

View File

@ -1,75 +0,0 @@
import { Injectable } from '@nestjs/common';
import { addMilliseconds } from 'date-fns';
import ms from 'ms';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
@Injectable()
export class QueryResultGettersFactory {
constructor(
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
) {}
async create<Result>(
result: Result,
objectMetadataItem: ObjectMetadataInterface,
): Promise<Result> {
// TODO: look for file type once implemented
switch (objectMetadataItem.nameSingular) {
case 'attachment':
return this.applyAttachmentGetters(result);
default:
return result;
}
}
private async applyAttachmentGetters<Result>(
attachments: any,
): Promise<Result> {
if (!attachments || !attachments.edges) {
return attachments;
}
const fileTokenExpiresIn = this.environmentService.get(
'FILE_TOKEN_EXPIRES_IN',
);
const secret = this.environmentService.get('FILE_TOKEN_SECRET');
const mappedEdges = await Promise.all(
attachments.edges.map(async (attachment: any) => {
if (!attachment.node.id || !attachment?.node?.fullPath) {
return attachment;
}
const expirationDate = addMilliseconds(
new Date(),
ms(fileTokenExpiresIn),
);
const signedPayload = await this.tokenService.encodePayload(
{
expiration_date: expirationDate,
attachment_id: attachment.node.id,
},
{
secret,
},
);
attachment.node.fullPath = `${attachment.node.fullPath}?token=${signedPayload}`;
return attachment;
}),
);
return {
...attachments,
edges: mappedEdges,
} as Result;
}
}

View File

@ -0,0 +1,45 @@
import { addMilliseconds } from 'date-fns';
import ms from 'ms';
import { QueryResultGuetterHandlerInterface } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface';
import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
export class AttachmentQueryResultGetterHandler
implements QueryResultGuetterHandlerInterface
{
constructor(
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
) {}
async process(attachment: any, workspaceId: string): Promise<any> {
if (!attachment.id || !attachment?.fullPath) {
return attachment;
}
const fileTokenExpiresIn = this.environmentService.get(
'FILE_TOKEN_EXPIRES_IN',
);
const secret = this.environmentService.get('FILE_TOKEN_SECRET');
const expirationDate = addMilliseconds(new Date(), ms(fileTokenExpiresIn));
const signedPayload = await this.tokenService.encodePayload(
{
expiration_date: expirationDate,
attachment_id: attachment.id,
workspace_id: workspaceId,
},
{
secret,
},
);
return {
...attachment,
fullPath: `${attachment.fullPath}?token=${signedPayload}`,
};
}
}

View File

@ -0,0 +1,3 @@
export interface QueryResultGuetterHandlerInterface {
process(result: any, workspaceId: string): Promise<any>;
}

View File

@ -0,0 +1,73 @@
import { Injectable } from '@nestjs/common';
import { QueryResultGuetterHandlerInterface } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/interfaces/query-result-getter-handler.interface';
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { AttachmentQueryResultGetterHandler } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/handlers/attachment-query-result-getter.handler';
import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
@Injectable()
export class QueryResultGettersFactory {
private handlers: Map<string, QueryResultGuetterHandlerInterface>;
constructor(
private readonly tokenService: TokenService,
private readonly environmentService: EnvironmentService,
) {
this.initializeHandlers();
}
private initializeHandlers() {
this.handlers = new Map<string, QueryResultGuetterHandlerInterface>([
[
'attachment',
new AttachmentQueryResultGetterHandler(
this.tokenService,
this.environmentService,
),
],
]);
}
async create(
result: any,
objectMetadataItem: ObjectMetadataInterface,
workspaceId: string,
): Promise<any> {
const handler = this.getHandler(objectMetadataItem.nameSingular);
if (result.edges) {
return {
...result,
edges: await Promise.all(
result.edges.map(async (edge: any) => ({
...edge,
node: await handler.process(edge.node, workspaceId),
})),
),
};
}
if (result.records) {
return {
...result,
records: await Promise.all(
result.records.map(
async (item: any) => await handler.process(item, workspaceId),
),
),
};
}
return await handler.process(result, workspaceId);
}
private getHandler(objectType: string): QueryResultGuetterHandlerInterface {
return (
this.handlers.get(objectType) || {
process: (result: any) => result,
}
);
}
}

View File

@ -25,7 +25,7 @@ import {
import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
import { WorkspaceQueryBuilderFactory } from 'src/engine/api/graphql/workspace-query-builder/workspace-query-builder.factory';
import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters.factory';
import { QueryResultGettersFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-result-getters/query-result-getters.factory';
import { QueryRunnerArgsFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/query-runner-args.factory';
import {
CallWebhookJobsJob,
@ -125,6 +125,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'',
workspaceId,
);
}
@ -167,6 +168,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'',
workspaceId,
);
return parsedResult?.edges?.[0]?.node;
@ -235,6 +237,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'',
workspaceId,
true,
);
}
@ -283,6 +286,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'insertInto',
workspaceId,
)
)?.records;
@ -418,6 +422,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'update',
workspaceId,
)
)?.records;
@ -485,6 +490,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'update',
workspaceId,
)
)?.records;
@ -555,6 +561,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'deleteFrom',
workspaceId,
)
)?.records;
@ -618,6 +625,7 @@ export class WorkspaceQueryRunnerService {
result,
objectMetadataItem,
'deleteFrom',
workspaceId,
)
)?.records;
@ -721,6 +729,7 @@ export class WorkspaceQueryRunnerService {
graphqlResult: PGGraphQLResult | undefined,
objectMetadataItem: ObjectMetadataInterface,
command: string,
workspaceId: string,
isMultiQuery = false,
): Promise<Result> {
const entityKey = `${command}${computeObjectTargetTable(
@ -767,6 +776,7 @@ export class WorkspaceQueryRunnerService {
const resultWithGetters = await this.queryResultGettersFactory.create(
result,
objectMetadataItem,
workspaceId,
);
return parseResult(resultWithGetters);
@ -780,7 +790,7 @@ export class WorkspaceQueryRunnerService {
): Promise<Result> {
const result = await this.execute(query, workspaceId);
return this.parseResult(result, objectMetadataItem, command);
return this.parseResult(result, objectMetadataItem, command, workspaceId);
}
async triggerWebhooks<Record>(

View File

@ -75,11 +75,6 @@ export class SignInUpService {
const passwordHash = password ? await hashPassword(password) : undefined;
let imagePath: string | undefined;
if (picture) {
imagePath = await this.uploadPicture(picture);
}
const existingUser = await this.userRepository.findOne({
where: {
email: email,
@ -103,7 +98,7 @@ export class SignInUpService {
workspaceInviteHash,
firstName,
lastName,
imagePath,
picture,
existingUser,
});
}
@ -113,7 +108,7 @@ export class SignInUpService {
passwordHash,
firstName,
lastName,
imagePath,
picture,
});
}
@ -126,7 +121,7 @@ export class SignInUpService {
workspaceInviteHash,
firstName,
lastName,
imagePath,
picture,
existingUser,
}: {
email: string;
@ -134,7 +129,7 @@ export class SignInUpService {
workspaceInviteHash: string;
firstName: string;
lastName: string;
imagePath: string | undefined;
picture: SignInUpServiceInput['picture'];
existingUser: User | null;
}) {
const workspace = await this.workspaceRepository.findOneBy({
@ -162,6 +157,8 @@ export class SignInUpService {
return Object.assign(existingUser, updatedUser);
}
const imagePath = await this.uploadPicture(picture, workspace.id);
const userToCreate = this.userRepository.create({
email: email,
firstName: firstName,
@ -185,13 +182,13 @@ export class SignInUpService {
passwordHash,
firstName,
lastName,
imagePath,
picture,
}: {
email: string;
passwordHash: string | undefined;
firstName: string;
lastName: string;
imagePath: string | undefined;
picture: SignInUpServiceInput['picture'];
}) {
assert(
!this.environmentService.get('IS_SIGN_UP_DISABLED'),
@ -208,6 +205,8 @@ export class SignInUpService {
const workspace = await this.workspaceRepository.save(workspaceToCreate);
const imagePath = await this.uploadPicture(picture, workspace.id);
const userToCreate = this.userRepository.create({
email: email,
firstName: firstName,
@ -225,7 +224,14 @@ export class SignInUpService {
return user;
}
async uploadPicture(picture: string): Promise<string> {
async uploadPicture(
picture: string | null | undefined,
workspaceId: string,
): Promise<string | undefined> {
if (!picture) {
return;
}
const buffer = await getImageBufferFromUrl(
picture,
this.httpService.axiosRef,
@ -238,6 +244,7 @@ export class SignInUpService {
filename: `${v4()}.${type?.ext}`,
mimeType: type?.mime,
fileFolder: FileFolder.ProfilePicture,
workspaceId,
});
return paths[0];

View File

@ -1,12 +1,17 @@
import { Controller, Get, Param, Res, UseGuards } from '@nestjs/common';
import { Controller, Get, Param, Req, Res, UseGuards } from '@nestjs/common';
import { Response } from 'express';
import { FilePathGuard } from 'src/engine/core-modules/file/guards/file-path-guard';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception';
import {
checkFilePath,
checkFilename,
} from 'src/engine/core-modules/file/file.utils';
import { FilePathGuard } from 'src/engine/core-modules/file/guards/file-path-guard';
import { FileService } from 'src/engine/core-modules/file/services/file.service';
// TODO: Add cookie authentication
@ -15,23 +20,38 @@ import { FileService } from 'src/engine/core-modules/file/services/file.service'
export class FileController {
constructor(private readonly fileService: FileService) {}
/**
* Serve files from local storage
* We recommend using an s3 bucket for production
*/
@Get('*/:filename')
async getFile(@Param() params: string[], @Res() res: Response) {
async getFile(
@Param() params: string[],
@Res() res: Response,
@Req() req: Request,
) {
const folderPath = checkFilePath(params[0]);
const filename = checkFilename(params['filename']);
const fileStream = await this.fileService.getFileStream(
folderPath,
filename,
);
fileStream.on('error', () => {
res.status(404).send({ error: 'File not found' });
});
const workspaceId = (req as any)?.workspaceId;
fileStream.pipe(res);
if (!workspaceId) {
return res.status(401).send({ error: 'Unauthorized' });
}
try {
const fileStream = await this.fileService.getFileStream(
folderPath,
filename,
workspaceId,
);
fileStream.pipe(res);
} catch (error) {
if (
error instanceof FileStorageException &&
error.code === FileStorageExceptionCode.FILE_NOT_FOUND
) {
return res.status(404).send({ error: 'File not found' });
}
return res.status(500).send({ error: 'Internal server error' });
}
}
}

View File

@ -1,14 +1,16 @@
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import { UseGuards } from '@nestjs/common';
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import { GraphQLUpload, FileUpload } from 'graphql-upload';
import { FileUpload, GraphQLUpload } from 'graphql-upload';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
@UseGuards(JwtAuthGuard, DemoEnvGuard)
@Resolver()
@ -17,6 +19,7 @@ export class FileUploadResolver {
@Mutation(() => String)
async uploadFile(
@AuthWorkspace() { id: workspaceId }: Workspace,
@Args({ name: 'file', type: () => GraphQLUpload })
{ createReadStream, filename, mimetype }: FileUpload,
@Args('fileFolder', { type: () => FileFolder, nullable: true })
@ -30,6 +33,7 @@ export class FileUploadResolver {
filename,
mimeType: mimetype,
fileFolder,
workspaceId,
});
return path;
@ -37,6 +41,7 @@ export class FileUploadResolver {
@Mutation(() => String)
async uploadImage(
@AuthWorkspace() { id: workspaceId }: Workspace,
@Args({ name: 'file', type: () => GraphQLUpload })
{ createReadStream, filename, mimetype }: FileUpload,
@Args('fileFolder', { type: () => FileFolder, nullable: true })
@ -50,6 +55,7 @@ export class FileUploadResolver {
filename,
mimeType: mimetype,
fileFolder,
workspaceId,
});
return paths[0];

View File

@ -1,15 +1,15 @@
import { Injectable } from '@nestjs/common';
import DOMPurify from 'dompurify';
import { JSDOM } from 'jsdom';
import sharp from 'sharp';
import { v4 as uuidV4 } from 'uuid';
import { JSDOM } from 'jsdom';
import DOMPurify from 'dompurify';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { getCropSize } from 'src/utils/image';
import { settings } from 'src/engine/constants/settings';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { getCropSize } from 'src/utils/image';
@Injectable()
export class FileUploadService {
@ -19,18 +19,18 @@ export class FileUploadService {
file,
filename,
mimeType,
fileFolder,
folder,
}: {
file: Buffer | Uint8Array | string;
filename: string;
mimeType: string | undefined;
fileFolder: FileFolder;
folder: string;
}) {
await this.fileStorage.write({
file,
name: filename,
mimeType,
folder: fileFolder,
folder,
});
}
@ -58,21 +58,24 @@ export class FileUploadService {
filename,
mimeType,
fileFolder,
workspaceId,
}: {
file: Buffer | Uint8Array | string;
filename: string;
mimeType: string | undefined;
fileFolder: FileFolder;
workspaceId: string;
}) {
const ext = filename.split('.')?.[1];
const id = uuidV4();
const name = `${id}${ext ? `.${ext}` : ''}`;
const folder = this.getWorkspaceFolderName(workspaceId, fileFolder);
await this._uploadFile({
file: this._sanitizeFile({ file, ext, mimeType }),
filename: name,
mimeType,
fileFolder,
folder,
});
return {
@ -87,11 +90,13 @@ export class FileUploadService {
filename,
mimeType,
fileFolder,
workspaceId,
}: {
file: Buffer | Uint8Array | string;
filename: string;
mimeType: string | undefined;
fileFolder: FileFolder;
workspaceId: string;
}) {
const ext = filename.split('.')?.[1];
const id = uuidV4();
@ -117,6 +122,7 @@ export class FileUploadService {
await Promise.all(
images.map(async (image, index) => {
const buffer = await image.toBuffer();
const folder = this.getWorkspaceFolderName(workspaceId, fileFolder);
paths.push(`${fileFolder}/${cropSizes[index]}/${name}`);
@ -124,7 +130,7 @@ export class FileUploadService {
file: buffer,
filename: `${cropSizes[index]}/${name}`,
mimeType,
fileFolder,
folder,
});
}),
);
@ -135,4 +141,8 @@ export class FileUploadService {
paths,
};
}
private getWorkspaceFolderName(workspaceId: string, fileFolder: FileFolder) {
return `workspace-${workspaceId}/${fileFolder}`;
}
}

View File

@ -0,0 +1,61 @@
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import {
checkFilename,
checkFilePath,
} from 'src/engine/core-modules/file/file.utils';
describe('FileUtils', () => {
describe('checkFilePath', () => {
it('should return sanitized file path', () => {
const filePath = `${FileFolder.Attachment}\0`;
const sanitizedFilePath = checkFilePath(filePath);
expect(sanitizedFilePath).toBe(`${FileFolder.Attachment}`);
});
it('should return sanitized file path with size', () => {
const filePath = `${FileFolder.ProfilePicture}\0/original`;
const sanitizedFilePath = checkFilePath(filePath);
expect(sanitizedFilePath).toBe(`${FileFolder.ProfilePicture}/original`);
});
it('should throw an error for invalid image size', () => {
const filePath = `${FileFolder.ProfilePicture}\0/invalid-size`;
expect(() => checkFilePath(filePath)).toThrow(
`Size invalid-size is not allowed`,
);
});
it('should throw an error for invalid folder', () => {
const filePath = `invalid-folder`;
expect(() => checkFilePath(filePath)).toThrow(
`Folder invalid-folder is not allowed`,
);
});
});
describe('checkFilename', () => {
it('should return sanitized filename', () => {
const filename = `${FileFolder.Attachment}\0.png`;
const sanitizedFilename = checkFilename(filename);
expect(sanitizedFilename).toBe(`${FileFolder.Attachment}.png`);
});
it('should throw an error for invalid filename', () => {
const filename = `invalid-filename`;
expect(() => checkFilename(filename)).toThrow(`Filename is not allowed`);
});
it('should throw an error for invalid filename', () => {
const filename = `\0`;
expect(() => checkFilename(filename)).toThrow(`Filename is not allowed`);
});
});
});

View File

@ -4,8 +4,8 @@ import { basename } from 'path';
import { KebabCase } from 'type-fest';
import { kebabCase } from 'src/utils/kebab-case';
import { settings } from 'src/engine/constants/settings';
import { kebabCase } from 'src/utils/kebab-case';
import { FileFolder } from './interfaces/file-folder.interface';

View File

@ -1,13 +1,13 @@
import {
Injectable,
CanActivate,
ExecutionContext,
HttpException,
HttpStatus,
Injectable,
} from '@nestjs/common';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { TokenService } from 'src/engine/core-modules/auth/services/token.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
@Injectable()
export class FilePathGuard implements CanActivate {
@ -17,25 +17,34 @@ export class FilePathGuard implements CanActivate {
) {}
async canActivate(context: ExecutionContext): Promise<boolean> {
const query = context.switchToHttp().getRequest().query;
const request = context.switchToHttp().getRequest();
const query = request.query;
if (query && query['token']) {
return !(await this.isExpired(query['token']));
const payloadToDecode = query['token'];
const decodedPayload = await this.tokenService.decodePayload(
payloadToDecode,
{
secret: this.environmentService.get('FILE_TOKEN_SECRET'),
},
);
const expirationDate = decodedPayload?.['expiration_date'];
const workspaceId = decodedPayload?.['workspace_id'];
const isExpired = await this.isExpired(expirationDate);
if (isExpired) {
return false;
}
request.workspaceId = workspaceId;
}
return true;
}
private async isExpired(signedExpirationDate: string): Promise<boolean> {
const decodedPayload = await this.tokenService.decodePayload(
signedExpirationDate,
{
secret: this.environmentService.get('FILE_TOKEN_SECRET'),
},
);
const expirationDate = decodedPayload?.['expiration_date'];
private async isExpired(expirationDate: string): Promise<boolean> {
if (!expirationDate) {
return true;
}

View File

@ -1,27 +0,0 @@
import { Test, TestingModule } from '@nestjs/testing';
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
import { FileUploadResolver } from './file-upload.resolver';
describe('FileUploadResolver', () => {
let resolver: FileUploadResolver;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
FileUploadResolver,
{
provide: FileUploadService,
useValue: {},
},
],
}).compile();
resolver = module.get<FileUploadResolver>(FileUploadResolver);
});
it('should be defined', () => {
expect(resolver).toBeDefined();
});
});

View File

@ -1,57 +0,0 @@
import { Args, Mutation, Resolver } from '@nestjs/graphql';
import { UseGuards } from '@nestjs/common';
import { GraphQLUpload, FileUpload } from 'graphql-upload';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
@UseGuards(JwtAuthGuard, DemoEnvGuard)
@Resolver()
export class FileUploadResolver {
constructor(private readonly fileUploadService: FileUploadService) {}
@Mutation(() => String)
async uploadFile(
@Args({ name: 'file', type: () => GraphQLUpload })
{ createReadStream, filename, mimetype }: FileUpload,
@Args('fileFolder', { type: () => FileFolder, nullable: true })
fileFolder: FileFolder,
): Promise<string> {
const stream = createReadStream();
const buffer = await streamToBuffer(stream);
const { path } = await this.fileUploadService.uploadFile({
file: buffer,
filename,
mimeType: mimetype,
fileFolder,
});
return path;
}
@Mutation(() => String)
async uploadImage(
@Args({ name: 'file', type: () => GraphQLUpload })
{ createReadStream, filename, mimetype }: FileUpload,
@Args('fileFolder', { type: () => FileFolder, nullable: true })
fileFolder: FileFolder,
): Promise<string> {
const stream = createReadStream();
const buffer = await streamToBuffer(stream);
const { paths } = await this.fileUploadService.uploadImage({
file: buffer,
filename,
mimeType: mimetype,
fileFolder,
});
return paths[0];
}
}

View File

@ -1,15 +1,42 @@
import { Injectable } from '@nestjs/common';
import { Stream } from 'stream';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
@Injectable()
export class FileService {
constructor(private readonly fileStorageService: FileStorageService) {}
async getFileStream(folderPath: string, filename: string) {
return this.fileStorageService.read({
folderPath,
filename,
});
async getFileStream(
folderPath: string,
filename: string,
workspaceId: string,
): Promise<Stream> {
const workspaceFolderPath = `workspace-${workspaceId}/${folderPath}`;
try {
return await this.fileStorageService.read({
folderPath: workspaceFolderPath,
filename,
});
} catch (error) {
// TODO: Remove this fallback when all files are moved to workspace folders
if (
error instanceof FileStorageException &&
error.code === FileStorageExceptionCode.FILE_NOT_FOUND
) {
return await this.fileStorageService.read({
folderPath,
filename,
});
}
throw error;
}
}
}

View File

@ -9,6 +9,7 @@ import {
} from '@nestjs/graphql';
import { InjectRepository } from '@nestjs/typeorm';
import assert from 'assert';
import crypto from 'crypto';
import { GraphQLJSONObject } from 'graphql-type-json';
@ -32,7 +33,6 @@ import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context';
import { assert } from 'src/utils/assert';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
const getHMACKey = (email?: string, key?: string | null) => {
@ -117,6 +117,7 @@ export class UserResolver {
@Mutation(() => String)
async uploadProfilePicture(
@AuthUser() { id }: User,
@AuthWorkspace() { id: workspaceId }: Workspace,
@Args({ name: 'file', type: () => GraphQLUpload })
{ createReadStream, filename, mimetype }: FileUpload,
): Promise<string> {
@ -133,6 +134,7 @@ export class UserResolver {
filename,
mimeType: mimetype,
fileFolder,
workspaceId,
});
return paths[0];

View File

@ -85,6 +85,7 @@ export class WorkspaceResolver {
filename,
mimeType: mimetype,
fileFolder,
workspaceId: id,
});
await this.workspaceService.updateOne(id, {

View File

@ -9,4 +9,8 @@ export interface StorageDriver {
folder: string;
mimeType: string | undefined;
}): Promise<void>;
move(params: {
from: { folderPath: string; filename: string };
to: { folderPath: string; filename: string };
}): Promise<void>;
}

View File

@ -1,8 +1,13 @@
import * as fs from 'fs/promises';
import { createReadStream, existsSync } from 'fs';
import { join, dirname } from 'path';
import * as fs from 'fs/promises';
import { dirname, join } from 'path';
import { Readable } from 'stream';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception';
import { StorageDriver } from './interfaces/storage-driver.interface';
export interface LocalDriverOptions {
@ -65,6 +70,49 @@ export class LocalDriver implements StorageDriver {
params.filename,
);
return createReadStream(filePath);
try {
return createReadStream(filePath);
} catch (error) {
if (error.code === 'ENOENT') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
throw error;
}
}
async move(params: {
from: { folderPath: string; filename: string };
to: { folderPath: string; filename: string };
}): Promise<void> {
const fromPath = join(
`${this.options.storagePath}/`,
params.from.folderPath,
params.from.filename,
);
const toPath = join(
`${this.options.storagePath}/`,
params.to.folderPath,
params.to.filename,
);
await this.createFolder(dirname(toPath));
try {
await fs.rename(fromPath, toPath);
} catch (error) {
if (error.code === 'ENOENT') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
throw error;
}
}
}

View File

@ -1,11 +1,13 @@
import { Readable } from 'stream';
import {
CopyObjectCommand,
CreateBucketCommandInput,
DeleteObjectCommand,
DeleteObjectsCommand,
GetObjectCommand,
HeadBucketCommandInput,
HeadObjectCommand,
ListObjectsV2Command,
NotFound,
PutObjectCommand,
@ -13,6 +15,11 @@ import {
S3ClientConfig,
} from '@aws-sdk/client-s3';
import {
FileStorageException,
FileStorageExceptionCode,
} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception';
import { StorageDriver } from './interfaces/storage-driver.interface';
export interface S3DriverOptions extends S3ClientConfig {
@ -115,13 +122,69 @@ export class S3Driver implements StorageDriver {
Key: `${params.folderPath}/${params.filename}`,
Bucket: this.bucketName,
});
const file = await this.s3Client.send(command);
if (!file || !file.Body || !(file.Body instanceof Readable)) {
throw new Error('Unable to get file stream');
try {
const file = await this.s3Client.send(command);
if (!file || !file.Body || !(file.Body instanceof Readable)) {
throw new Error('Unable to get file stream');
}
return Readable.from(file.Body);
} catch (error) {
if (error.name === 'NoSuchKey') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
throw error;
}
}
return Readable.from(file.Body);
async move(params: {
from: { folderPath: string; filename: string };
to: { folderPath: string; filename: string };
}): Promise<void> {
const fromKey = `${params.from.folderPath}/${params.from.filename}`;
const toKey = `${params.to.folderPath}/${params.to.filename}`;
try {
// Check if the source file exists
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: fromKey,
}),
);
// Copy the object to the new location
await this.s3Client.send(
new CopyObjectCommand({
CopySource: `${this.bucketName}/${fromKey}`,
Bucket: this.bucketName,
Key: toKey,
}),
);
// Delete the original object
await this.s3Client.send(
new DeleteObjectCommand({
Bucket: this.bucketName,
Key: fromKey,
}),
);
} catch (error) {
if (error.name === 'NotFound') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
// For other errors, throw the original error
throw error;
}
}
async checkBucketExists(args: HeadBucketCommandInput) {

View File

@ -26,4 +26,11 @@ export class FileStorageService implements StorageDriver {
read(params: { folderPath: string; filename: string }): Promise<Readable> {
return this.driver.read(params);
}
move(params: {
from: { folderPath: string; filename: string };
to: { folderPath: string; filename: string };
}): Promise<void> {
return this.driver.move(params);
}
}

View File

@ -0,0 +1,12 @@
import { CustomException } from 'src/utils/custom-exception';
export class FileStorageException extends CustomException {
code: FileStorageExceptionCode;
constructor(message: string, code: FileStorageExceptionCode) {
super(message, code);
}
}
export enum FileStorageExceptionCode {
FILE_NOT_FOUND = 'FILE_NOT_FOUND',
}