feat: upload module (#486)

* feat: wip upload module

* feat: local storage and serve local images

* feat: protect against injections

* feat: server local and s3 files

* fix: use storage location when serving local files

* feat: cross field env validation

server/src/core/auth/auth.module.ts

@@ -1,6 +1,5 @@
 import { Module } from '@nestjs/common';
 import { JwtModule } from '@nestjs/jwt';
-import { ConfigModule, ConfigService } from '@nestjs/config';
 import { JwtAuthStrategy } from './strategies/jwt.auth.strategy';
 import { AuthService } from './services/auth.service';
 import { GoogleAuthController } from './controllers/google-auth.controller';
@@ -8,25 +7,24 @@ import { GoogleStrategy } from './strategies/google.auth.strategy';
 import { PrismaService } from 'src/database/prisma.service';
 import { UserModule } from '../user/user.module';
 import { VerifyAuthController } from './controllers/verify-auth.controller';
-
 import { TokenService } from './services/token.service';
 import { AuthResolver } from './auth.resolver';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
 
 const jwtModule = JwtModule.registerAsync({
-  useFactory: async (configService: ConfigService) => {
+  useFactory: async (environmentService: EnvironmentService) => {
     return {
-      secret: configService.get<string>('ACCESS_TOKEN_SECRET'),
+      secret: environmentService.getAccessTokenSecret(),
       signOptions: {
-        expiresIn: configService.get<string>('ACCESS_TOKEN_EXPIRES_IN'),
+        expiresIn: environmentService.getAccessTokenExpiresIn(),
       },
     };
   },
-  imports: [ConfigModule.forRoot({})],
-  inject: [ConfigService],
+  inject: [EnvironmentService],
 });
 
 @Module({
-  imports: [jwtModule, ConfigModule.forRoot({}), UserModule],
+  imports: [jwtModule, UserModule],
   controllers: [GoogleAuthController, VerifyAuthController],
   providers: [
     AuthService,

server/src/core/auth/services/token.service.spec.ts

@@ -3,7 +3,7 @@ import { TokenService } from './token.service';
 import { PrismaService } from 'src/database/prisma.service';
 import { prismaMock } from 'src/database/client-mock/jest-prisma-singleton';
 import { JwtService } from '@nestjs/jwt';
-import { ConfigService } from '@nestjs/config';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
 
 describe('TokenService', () => {
   let service: TokenService;
@@ -17,7 +17,7 @@ describe('TokenService', () => {
           useValue: {},
         },
         {
-          provide: ConfigService,
+          provide: EnvironmentService,
           useValue: {},
         },
         {

server/src/core/auth/services/token.service.ts

@@ -8,24 +8,24 @@ import {
 } from '@nestjs/common';
 import { JwtService } from '@nestjs/jwt';
 import { JwtPayload } from '../strategies/jwt.auth.strategy';
-import { ConfigService } from '@nestjs/config';
 import { PrismaService } from 'src/database/prisma.service';
 import { assert } from 'src/utils/assert';
 import { addMilliseconds } from 'date-fns';
 import ms from 'ms';
 import { AuthToken } from '../dto/token.entity';
 import { TokenExpiredError } from 'jsonwebtoken';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
 
 @Injectable()
 export class TokenService {
   constructor(
     private readonly jwtService: JwtService,
-    private readonly configService: ConfigService,
+    private readonly environmentService: EnvironmentService,
     private readonly prismaService: PrismaService,
   ) {}
 
   async generateAccessToken(userId: string): Promise<AuthToken> {
-    const expiresIn = this.configService.get<string>('ACCESS_TOKEN_EXPIRES_IN');
+    const expiresIn = this.environmentService.getAccessTokenExpiresIn();
     assert(expiresIn, '', InternalServerErrorException);
     const expiresAt = addMilliseconds(new Date().getTime(), ms(expiresIn));
 
@@ -56,10 +56,8 @@ export class TokenService {
   }
 
   async generateRefreshToken(userId: string): Promise<AuthToken> {
-    const secret = this.configService.get('REFRESH_TOKEN_SECRET');
-    const expiresIn = this.configService.get<string>(
-      'REFRESH_TOKEN_EXPIRES_IN',
-    );
+    const secret = this.environmentService.getRefreshTokenSecret();
+    const expiresIn = this.environmentService.getRefreshTokenExpiresIn();
     assert(expiresIn, '', InternalServerErrorException);
     const expiresAt = addMilliseconds(new Date().getTime(), ms(expiresIn));
 
@@ -87,8 +85,8 @@ export class TokenService {
   }
 
   async generateLoginToken(email: string): Promise<AuthToken> {
-    const secret = this.configService.get('LOGIN_TOKEN_SECRET');
-    const expiresIn = this.configService.get<string>('LOGIN_TOKEN_EXPIRES_IN');
+    const secret = this.environmentService.getLoginTokenSecret();
+    const expiresIn = this.environmentService.getLoginTokenExpiresIn();
     assert(expiresIn, '', InternalServerErrorException);
     const expiresAt = addMilliseconds(new Date().getTime(), ms(expiresIn));
     const jwtPayload = {
@@ -105,7 +103,7 @@ export class TokenService {
   }
 
   async verifyLoginToken(loginToken: string): Promise<string> {
-    const loginTokenSecret = this.configService.get('LOGIN_TOKEN_SECRET');
+    const loginTokenSecret = this.environmentService.getLoginTokenSecret();
 
     const payload = await this.verifyJwt(loginToken, loginTokenSecret);
 
@@ -113,7 +111,7 @@ export class TokenService {
   }
 
   async verifyRefreshToken(refreshToken: string) {
-    const secret = this.configService.get('REFRESH_TOKEN_SECRET');
+    const secret = this.environmentService.getRefreshTokenSecret();
     const jwtPayload = await this.verifyJwt(refreshToken, secret);
 
     assert(
@@ -191,9 +189,7 @@ export class TokenService {
   }
 
   computeRedirectURI(loginToken: string): string {
-    return `${this.configService.get<string>(
-      'FRONT_AUTH_CALLBACK_URL',
-    )}?loginToken=${loginToken}`;
+    return `${this.environmentService.getFrontAuthCallbackUrl()}?loginToken=${loginToken}`;
   }
 
   async verifyJwt(token: string, secret?: string) {

server/src/core/auth/strategies/google.auth.strategy.ts

@@ -2,8 +2,8 @@ import { PassportStrategy } from '@nestjs/passport';
 import { Strategy, VerifyCallback } from 'passport-google-oauth20';
 
 import { Injectable } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
 import { Request } from 'express';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
 
 export type GoogleRequest = Request & {
   user: {
@@ -15,11 +15,11 @@ export type GoogleRequest = Request & {
 
 @Injectable()
 export class GoogleStrategy extends PassportStrategy(Strategy, 'google') {
-  constructor(configService: ConfigService) {
+  constructor(environmentService: EnvironmentService) {
     super({
-      clientID: configService.get<string>('AUTH_GOOGLE_CLIENT_ID'),
-      clientSecret: configService.get<string>('AUTH_GOOGLE_CLIENT_SECRET'),
-      callbackURL: configService.get<string>('AUTH_GOOGLE_CALLBACK_URL'),
+      clientID: environmentService.getAuthGoogleClientId(),
+      clientSecret: environmentService.getAuthGoogleClientSecret(),
+      callbackURL: environmentService.getAuthGoogleCallbackUrl(),
       scope: ['email', 'profile'],
     });
   }

server/src/core/auth/strategies/jwt.auth.strategy.ts

@@ -1,9 +1,9 @@
 import { Strategy, ExtractJwt } from 'passport-jwt';
 import { PassportStrategy } from '@nestjs/passport';
 import { Injectable, UnauthorizedException } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
 import { PrismaService } from 'src/database/prisma.service';
 import { User, Workspace } from '@prisma/client';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
 
 export type JwtPayload = { sub: string; workspaceId: string };
 export type PassportUser = { user: User; workspace: Workspace };
@@ -11,13 +11,13 @@ export type PassportUser = { user: User; workspace: Workspace };
 @Injectable()
 export class JwtAuthStrategy extends PassportStrategy(Strategy, 'jwt') {
   constructor(
-    private readonly configService: ConfigService,
+    private readonly environmentService: EnvironmentService,
     private readonly prismaService: PrismaService,
   ) {
     super({
       jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
       ignoreExpiration: false,
-      secretOrKey: configService.get<string>('ACCESS_TOKEN_SECRET'),
+      secretOrKey: environmentService.getAccessTokenSecret(),
     });
   }
 

server/src/core/core.module.ts

@@ -7,6 +7,7 @@ import { PipelineModule } from './pipeline/pipeline.module';
 import { AuthModule } from './auth/auth.module';
 import { WorkspaceModule } from './workspace/workspace.module';
 import { AnalyticsModule } from './analytics/analytics.module';
+import { FileModule } from './file/file.module';
 
 @Module({
   imports: [
@@ -18,6 +19,7 @@ import { AnalyticsModule } from './analytics/analytics.module';
     PipelineModule,
     WorkspaceModule,
     AnalyticsModule,
+    FileModule,
   ],
   exports: [
     AuthModule,

server/src/core/file/controllers/file.controller.spec.ts

@@ -0,0 +1,25 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { FileController } from './file.controller';
+import { FileService } from '../services/file.service';
+
+describe('FileController', () => {
+  let controller: FileController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [FileController],
+      providers: [
+        {
+          provide: FileService,
+          useValue: {},
+        },
+      ],
+    }).compile();
+
+    controller = module.get<FileController>(FileController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

server/src/core/file/controllers/file.controller.ts

@@ -0,0 +1,30 @@
+import { Controller, Get, Param, Res, UseGuards } from '@nestjs/common';
+import { Response } from 'express';
+import { checkFilePath, checkFilename } from '../file.utils';
+import { FileService } from '../services/file.service';
+import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
+
+@UseGuards(JwtAuthGuard)
+@Controller('files')
+export class FileController {
+  constructor(private readonly fileService: FileService) {}
+  /**
+   * Serve files from local storage
+   * We recommend using an s3 bucket for production
+   */
+  @Get('*/:filename')
+  async getFile(@Param() params: string[], @Res() res: Response) {
+    const folderPath = checkFilePath(params[0]);
+    const filename = checkFilename(params['filename']);
+    const fileStream = await this.fileService.getFileStream(
+      folderPath,
+      filename,
+    );
+
+    fileStream.on('error', () => {
+      res.status(404).send({ error: 'File not found' });
+    });
+
+    fileStream.pipe(res);
+  }
+}

server/src/core/file/file.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+import { FileService } from './services/file.service';
+import { FileUploadService } from './services/file-upload.service';
+import { FileUploadResolver } from './resolvers/file-upload.resolver';
+import { FileController } from './controllers/file.controller';
+
+@Module({
+  providers: [FileService, FileUploadService, FileUploadResolver],
+  exports: [FileService, FileUploadService],
+  controllers: [FileController],
+})
+export class FileModule {}

server/src/core/file/file.utils.ts

@@ -0,0 +1,46 @@
+import { kebabCase } from 'src/utils/kebab-case';
+import { FileFolder } from './interfaces/file-folder.interface';
+import { KebabCase } from 'type-fest';
+import { BadRequestException } from '@nestjs/common';
+import { basename } from 'path';
+import { settings } from 'src/constants/settings';
+import { camelCase } from 'src/utils/camel-case';
+
+type AllowedFolders = KebabCase<keyof typeof FileFolder>;
+
+export function checkFilePath(filePath: string): string {
+  const allowedFolders = Object.values(FileFolder).map((value) =>
+    kebabCase(value),
+  );
+
+  const sanitizedFilePath = filePath.replace(/\0/g, '');
+  const [folder, size] = sanitizedFilePath.split('/');
+
+  if (!allowedFolders.includes(folder as AllowedFolders)) {
+    throw new BadRequestException(`Folder ${folder} is not allowed`);
+  }
+
+  if (
+    size &&
+    !settings.storage.imageCropSizes[camelCase(folder)]?.includes(size)
+  ) {
+    throw new BadRequestException(`Size ${size} is not allowed`);
+  }
+
+  return sanitizedFilePath;
+}
+
+export function checkFilename(filename: string) {
+  const sanitizedFilename = basename(filename.replace(/\0/g, ''));
+
+  if (
+    !sanitizedFilename ||
+    sanitizedFilename.includes('/') ||
+    sanitizedFilename.includes('\\') ||
+    !sanitizedFilename.includes('.')
+  ) {
+    throw new BadRequestException(`Filename is not allowed`);
+  }
+
+  return basename(sanitizedFilename);
+}

server/src/core/file/interfaces/file-folder.interface.ts

@@ -0,0 +1,9 @@
+import { registerEnumType } from '@nestjs/graphql';
+
+export enum FileFolder {
+  ProfilePicture = 'profilePicture',
+}
+
+registerEnumType(FileFolder, {
+  name: 'FileFolder',
+});

server/src/core/file/resolvers/file-upload.resolver.spec.ts

@@ -0,0 +1,25 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { FileUploadResolver } from './file-upload.resolver';
+import { FileUploadService } from '../services/file-upload.service';
+
+describe('FileUploadResolver', () => {
+  let resolver: FileUploadResolver;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        FileUploadResolver,
+        {
+          provide: FileUploadService,
+          useValue: {},
+        },
+      ],
+    }).compile();
+
+    resolver = module.get<FileUploadResolver>(FileUploadResolver);
+  });
+
+  it('should be defined', () => {
+    expect(resolver).toBeDefined();
+  });
+});

server/src/core/file/resolvers/file-upload.resolver.ts

@@ -0,0 +1,60 @@
+import { Args, Mutation, Resolver } from '@nestjs/graphql';
+import { GraphQLUpload, FileUpload } from 'graphql-upload';
+import { v4 as uuidV4 } from 'uuid';
+import { FileUploadService } from '../services/file-upload.service';
+import { UseGuards } from '@nestjs/common';
+import { JwtAuthGuard } from 'src/guards/jwt.auth.guard';
+import { streamToBuffer } from 'src/utils/stream-to-buffer';
+import { FileFolder } from '../interfaces/file-folder.interface';
+
+@UseGuards(JwtAuthGuard)
+@Resolver()
+export class FileUploadResolver {
+  constructor(private readonly fileUploadService: FileUploadService) {}
+
+  @Mutation(() => String)
+  async uploadFile(
+    @Args({ name: 'file', type: () => GraphQLUpload })
+    { createReadStream, filename, mimetype }: FileUpload,
+    @Args('fileFolder', { type: () => FileFolder, nullable: true })
+    fileFolder: FileFolder,
+  ): Promise<string> {
+    const stream = createReadStream();
+    const buffer = await streamToBuffer(stream);
+    const ext = filename.split('.')?.[1];
+    const id = uuidV4();
+    const name = `${id}${ext ? `.${ext}` : ''}`;
+
+    const path = await this.fileUploadService.uploadFile({
+      file: buffer,
+      name,
+      mimeType: mimetype,
+      fileFolder,
+    });
+
+    return path.name;
+  }
+
+  @Mutation(() => String)
+  async uploadImage(
+    @Args({ name: 'file', type: () => GraphQLUpload })
+    { createReadStream, filename, mimetype }: FileUpload,
+    @Args('fileFolder', { type: () => FileFolder, nullable: true })
+    fileFolder: FileFolder,
+  ): Promise<string> {
+    const stream = createReadStream();
+    const buffer = await streamToBuffer(stream);
+    const ext = filename.split('.')?.[1];
+    const id = uuidV4();
+    const name = `${id}${ext ? `.${ext}` : ''}`;
+
+    const path = await this.fileUploadService.uploadImage({
+      file: buffer,
+      name,
+      mimeType: mimetype,
+      fileFolder,
+    });
+
+    return path.name;
+  }
+}

server/src/core/file/services/file-upload.service.spec.ts

@@ -0,0 +1,35 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { FileUploadService } from './file-upload.service';
+import { S3StorageService } from 'src/integrations/s3-storage/s3-storage.service';
+import { LocalStorageService } from 'src/integrations/local-storage/local-storage.service';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+
+describe('FileUploadService', () => {
+  let service: FileUploadService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        FileUploadService,
+        {
+          provide: S3StorageService,
+          useValue: {},
+        },
+        {
+          provide: LocalStorageService,
+          useValue: {},
+        },
+        {
+          provide: EnvironmentService,
+          useValue: {},
+        },
+      ],
+    }).compile();
+
+    service = module.get<FileUploadService>(FileUploadService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

server/src/core/file/services/file-upload.service.ts

@@ -0,0 +1,135 @@
+import { Injectable } from '@nestjs/common';
+import sharp from 'sharp';
+import { S3StorageService } from 'src/integrations/s3-storage/s3-storage.service';
+import { kebabCase } from 'src/utils/kebab-case';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+import { LocalStorageService } from 'src/integrations/local-storage/local-storage.service';
+import { getCropSize } from 'src/utils/image';
+import { settings } from 'src/constants/settings';
+import { FileFolder } from '../interfaces/file-folder.interface';
+
+@Injectable()
+export class FileUploadService {
+  constructor(
+    private readonly s3Storage: S3StorageService,
+    private readonly localStorage: LocalStorageService,
+    private readonly environmentService: EnvironmentService,
+  ) {}
+
+  async uploadFile({
+    file,
+    name,
+    mimeType,
+    fileFolder,
+  }: {
+    file: Buffer | Uint8Array | string;
+    name: string;
+    mimeType: string | undefined;
+    fileFolder: FileFolder;
+  }) {
+    const storageType = this.environmentService.getStorageType();
+
+    switch (storageType) {
+      case 's3': {
+        await this.uploadFileToS3(file, name, mimeType, fileFolder);
+        return {
+          name: `/${name}`,
+        };
+      }
+      case 'local':
+      default: {
+        await this.uploadToLocal(file, name, fileFolder);
+        return {
+          name: `/${name}`,
+        };
+      }
+    }
+  }
+
+  async uploadImage({
+    file,
+    name,
+    mimeType,
+    fileFolder,
+  }: {
+    file: Buffer | Uint8Array | string;
+    name: string;
+    mimeType: string | undefined;
+    fileFolder: FileFolder;
+  }) {
+    // Get all cropSizes for this fileFolder
+    const cropSizes = settings.storage.imageCropSizes[fileFolder];
+    // Extract the values from ShortCropSize
+    const sizes = cropSizes.map((shortSize) => getCropSize(shortSize));
+    // Crop images based on sizes
+    const images = await Promise.all(
+      sizes.map((size) =>
+        sharp(file).resize({
+          [size?.type || 'width']: size?.value ?? undefined,
+        }),
+      ),
+    );
+
+    // Upload all images to corresponding folders
+    await Promise.all(
+      images.map(async (image, index) => {
+        const buffer = await image.toBuffer();
+
+        return this.uploadFile({
+          file: buffer,
+          name: `${cropSizes[index]}/${name}`,
+          mimeType,
+          fileFolder,
+        });
+      }),
+    );
+
+    return {
+      name: `/${name}`,
+    };
+  }
+
+  private async uploadToLocal(
+    file: Buffer | Uint8Array | string,
+    name: string,
+    fileFolder: FileFolder,
+  ): Promise<void> {
+    const folderName = kebabCase(fileFolder.toString());
+
+    try {
+      const result = await this.localStorage.uploadFile({
+        file,
+        name,
+        folder: folderName,
+      });
+
+      return result;
+    } catch (err) {
+      console.log('uploadFile error: ', err);
+      throw err;
+    }
+  }
+
+  private async uploadFileToS3(
+    file: Buffer | Uint8Array | string,
+    name: string,
+    mimeType: string | undefined,
+    fileFolder: FileFolder,
+  ) {
+    // Aws only accept bucket with kebab-case name
+    const bucketFolderName = kebabCase(fileFolder.toString());
+
+    try {
+      const result = await this.s3Storage.uploadFile({
+        Key: `${bucketFolderName}/${name}`,
+        Body: file,
+        ContentType: mimeType,
+      });
+
+      return result;
+    } catch (err) {
+      console.log('uploadFile error: ', err);
+      throw err;
+    }
+  }
+}

server/src/core/file/services/file.service.spec.ts

@@ -0,0 +1,30 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { FileService } from './file.service';
+import { S3StorageService } from 'src/integrations/s3-storage/s3-storage.service';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+
+describe('FileService', () => {
+  let service: FileService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        FileService,
+        {
+          provide: S3StorageService,
+          useValue: {},
+        },
+        {
+          provide: EnvironmentService,
+          useValue: {},
+        },
+      ],
+    }).compile();
+
+    service = module.get<FileService>(FileService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

server/src/core/file/services/file.service.ts

@@ -0,0 +1,55 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+import { S3StorageService } from 'src/integrations/s3-storage/s3-storage.service';
+import { EnvironmentService } from 'src/integrations/environment/environment.service';
+import { createReadStream } from 'fs';
+import { join } from 'path';
+import { Readable } from 'stream';
+
+@Injectable()
+export class FileService {
+  constructor(
+    private readonly s3Storage: S3StorageService,
+    private readonly environmentService: EnvironmentService,
+  ) {}
+
+  async getFileStream(folderPath: string, filename: string) {
+    const storageType = this.environmentService.getStorageType();
+
+    switch (storageType) {
+      case 's3':
+        return this.getS3FileStream(folderPath, filename);
+      case 'local':
+      default:
+        return this.getLocalFileStream(folderPath, filename);
+    }
+  }
+
+  private async getLocalFileStream(folderPath: string, filename: string) {
+    const storageLocation = this.environmentService.getStorageLocation();
+
+    const filePath = join(
+      process.cwd(),
+      `${storageLocation}/`,
+      folderPath,
+      filename,
+    );
+
+    return createReadStream(filePath);
+  }
+
+  private async getS3FileStream(folderPath: string, filename: string) {
+    try {
+      const file = await this.s3Storage.getFile({
+        Key: `${folderPath}/${filename}`,
+      });
+
+      if (!file || !file.Body || !(file.Body instanceof Readable)) {
+        throw new Error('Unable to get file stream');
+      }
+
+      return Readable.from(file.Body);
+    } catch (error) {
+      throw new NotFoundException('File not found');
+    }
+  }
+}