Serverless function improvements (#6769)

- add layer for lambda execution - add layer for local execution - add package resolve for the monaco editor - add route to get installed package for serverless functions - add layer versioning
2024-09-02 15:25:20 +02:00
parent f8890689ee
commit 7e03419c16
41 changed files with 4834 additions and 164 deletions
--- a/packages/twenty-server/src/engine/integrations/file-storage/drivers/local.driver.ts
+++ b/packages/twenty-server/src/engine/integrations/file-storage/drivers/local.driver.ts
@ -127,6 +127,9 @@ export class LocalDriver implements StorageDriver {
    from: { folderPath: string; filename?: string };
    to: { folderPath: string; filename?: string };
  }): Promise<void> {
+    if (!params.from.filename && params.to.filename) {
+      throw new Error('Cannot copy folder to file');
+    }
    const fromPath = join(
      `${this.options.storagePath}/`,
      params.from.folderPath,
@ -139,6 +142,8 @@ export class LocalDriver implements StorageDriver {
      params.to.filename || '',
    );

+    await this.createFolder(dirname(toPath));
+
    try {
      await fs.cp(fromPath, toPath, { recursive: true });
    } catch (error) {
--- a/packages/twenty-server/src/engine/integrations/file-storage/drivers/s3.driver.ts
+++ b/packages/twenty-server/src/engine/integrations/file-storage/drivers/s3.driver.ts
@ -20,6 +20,8 @@ import {
  FileStorageExceptionCode,
 } from 'src/engine/integrations/file-storage/interfaces/file-storage-exception';

+import { isDefined } from 'src/utils/is-defined';
+
 import { StorageDriver } from './interfaces/storage-driver.interface';

 export interface S3DriverOptions extends S3ClientConfig {
@ -191,35 +193,80 @@ export class S3Driver implements StorageDriver {
    from: { folderPath: string; filename?: string };
    to: { folderPath: string; filename?: string };
  }): Promise<void> {
+    if (!params.from.filename && params.to.filename) {
+      throw new Error('Cannot copy folder to file');
+    }
+
    const fromKey = `${params.from.folderPath}/${params.from.filename || ''}`;
    const toKey = `${params.to.folderPath}/${params.to.filename || ''}`;

-    try {
-      // Check if the source file exists
-      await this.s3Client.send(
-        new HeadObjectCommand({
-          Bucket: this.bucketName,
-          Key: fromKey,
-        }),
+    if (isDefined(params.from.filename)) {
+      try {
+        // Check if the source file exists
+        await this.s3Client.send(
+          new HeadObjectCommand({
+            Bucket: this.bucketName,
+            Key: fromKey,
+          }),
+        );
+
+        // Copy the object to the new location
+        await this.s3Client.send(
+          new CopyObjectCommand({
+            CopySource: `${this.bucketName}/${fromKey}`,
+            Bucket: this.bucketName,
+            Key: toKey,
+          }),
+        );
+
+        return;
+      } catch (error) {
+        if (error.name === 'NotFound') {
+          throw new FileStorageException(
+            'File not found',
+            FileStorageExceptionCode.FILE_NOT_FOUND,
+          );
+        }
+        // For other errors, throw the original error
+        throw error;
+      }
+    }
+
+    const listedObjects = await this.s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: this.bucketName,
+        Prefix: fromKey,
+      }),
+    );
+
+    if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
+      throw new Error('No objects found in the source folder.');
+    }
+
+    for (const object of listedObjects.Contents) {
+      const match = object.Key?.match(/(.*)\/(.*)/);
+
+      if (!isDefined(match)) {
+        continue;
+      }
+      const fromFolderPath = match[1];
+      const filename = match[2];
+      const toFolderPath = fromFolderPath.replace(
+        params.from.folderPath,
+        params.to.folderPath,
      );

-      // Copy the object to the new location
-      await this.s3Client.send(
-        new CopyObjectCommand({
-          CopySource: `${this.bucketName}/${fromKey}`,
-          Bucket: this.bucketName,
-          Key: toKey,
-        }),
-      );
-    } catch (error) {
-      if (error.name === 'NotFound') {
-        throw new FileStorageException(
-          'File not found',
-          FileStorageExceptionCode.FILE_NOT_FOUND,
-        );
+      if (!isDefined(toFolderPath)) {
+        continue;
      }
-      // For other errors, throw the original error
-      throw error;
+
+      await this.copy({
+        from: {
+          folderPath: fromFolderPath,
+          filename,
+        },
+        to: { folderPath: toFolderPath, filename },
+      });
    }
  }

--- a/packages/twenty-server/src/engine/integrations/integrations.module.ts
+++ b/packages/twenty-server/src/engine/integrations/integrations.module.ts
@ -17,7 +17,6 @@ import { LLMTracingModule } from 'src/engine/integrations/llm-tracing/llm-tracin
 import { llmTracingModuleFactory } from 'src/engine/integrations/llm-tracing/llm-tracing.module-factory';
 import { loggerModuleFactory } from 'src/engine/integrations/logger/logger.module-factory';
 import { messageQueueModuleFactory } from 'src/engine/integrations/message-queue/message-queue.module-factory';
-import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
 import { serverlessModuleFactory } from 'src/engine/integrations/serverless/serverless-module.factory';
 import { ServerlessModule } from 'src/engine/integrations/serverless/serverless.module';

@ -68,11 +67,7 @@ import { MessageQueueModule } from './message-queue/message-queue.module';
    }),
    ServerlessModule.forRootAsync({
      useFactory: serverlessModuleFactory,
-      inject: [
-        EnvironmentService,
-        FileStorageService,
-        BuildDirectoryManagerService,
-      ],
+      inject: [EnvironmentService, FileStorageService],
    }),
  ],
  exports: [],
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/constants/common-layer-name.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/constants/common-layer-name.ts
@ -0,0 +1 @@
+export const COMMON_LAYER_NAME = 'common-layer';
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/constants/serverless-tmpdir-folder.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/constants/serverless-tmpdir-folder.ts
@ -0,0 +1,4 @@
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+export const SERVERLESS_TMPDIR_FOLDER = join(tmpdir(), 'serverless-tmpdir');
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface.ts
@ -23,7 +23,7 @@ export interface ServerlessDriver {
  publish(serverlessFunction: ServerlessFunctionEntity): Promise<string>;
  execute(
    serverlessFunction: ServerlessFunctionEntity,
-    payload: object | undefined,
+    payload: object,
    version: string,
  ): Promise<ServerlessExecuteResult>;
 }
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/lambda.driver.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/lambda.driver.ts
@ -1,17 +1,23 @@
-import fs from 'fs';
+import * as fs from 'fs/promises';
+import { join } from 'path';

 import {
  CreateFunctionCommand,
  DeleteFunctionCommand,
  GetFunctionCommand,
  InvokeCommand,
+  InvokeCommandInput,
  Lambda,
  LambdaClientConfig,
+  PublishLayerVersionCommand,
+  PublishLayerVersionCommandInput,
  PublishVersionCommand,
  PublishVersionCommandInput,
  ResourceNotFoundException,
  UpdateFunctionCodeCommand,
  waitUntilFunctionUpdatedV2,
+  ListLayerVersionsCommandInput,
+  ListLayerVersionsCommand,
 } from '@aws-sdk/client-lambda';
 import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
 import { UpdateFunctionCodeCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/UpdateFunctionCodeCommand';
@ -21,20 +27,28 @@ import {
  ServerlessExecuteResult,
 } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';

+import {
+  ServerlessFunctionEntity,
+  ServerlessFunctionRuntime,
+} from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
+import {
+  LambdaBuildDirectoryManager,
+  NODE_LAYER_SUBFOLDER,
+} from 'src/engine/integrations/serverless/drivers/utils/lambda-build-directory-manager';
 import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
 import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
-import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
 import { createZipFile } from 'src/engine/integrations/serverless/drivers/utils/create-zip-file';
 import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
-import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
 import {
  ServerlessFunctionException,
  ServerlessFunctionExceptionCode,
 } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
+import { isDefined } from 'src/utils/is-defined';
+import { COMMON_LAYER_NAME } from 'src/engine/integrations/serverless/drivers/constants/common-layer-name';
+import { copyAndBuildDependencies } from 'src/engine/integrations/serverless/drivers/utils/copy-and-build-dependencies';

 export interface LambdaDriverOptions extends LambdaClientConfig {
  fileStorageService: FileStorageService;
-  buildDirectoryManagerService: BuildDirectoryManagerService;
  region: string;
  role: string;
 }
@ -46,7 +60,6 @@ export class LambdaDriver
  private readonly lambdaClient: Lambda;
  private readonly lambdaRole: string;
  private readonly fileStorageService: FileStorageService;
-  private readonly buildDirectoryManagerService: BuildDirectoryManagerService;

  constructor(options: LambdaDriverOptions) {
    super();
@ -55,7 +68,69 @@ export class LambdaDriver
    this.lambdaClient = new Lambda({ ...lambdaOptions, region });
    this.lambdaRole = role;
    this.fileStorageService = options.fileStorageService;
-    this.buildDirectoryManagerService = options.buildDirectoryManagerService;
+  }
+
+  private async waitFunctionUpdates(
+    serverlessFunctionId: string,
+    maxWaitTime: number,
+  ) {
+    const waitParams = { FunctionName: serverlessFunctionId };
+
+    await waitUntilFunctionUpdatedV2(
+      { client: this.lambdaClient, maxWaitTime },
+      waitParams,
+    );
+  }
+
+  private async createLayerIfNotExists(version: number): Promise<string> {
+    const listLayerParams: ListLayerVersionsCommandInput = {
+      LayerName: COMMON_LAYER_NAME,
+      MaxItems: 1,
+    };
+    const listLayerCommand = new ListLayerVersionsCommand(listLayerParams);
+    const listLayerResult = await this.lambdaClient.send(listLayerCommand);
+
+    if (
+      isDefined(listLayerResult.LayerVersions) &&
+      listLayerResult.LayerVersions?.[0].Description === `${version}` &&
+      isDefined(listLayerResult.LayerVersions[0].LayerVersionArn)
+    ) {
+      return listLayerResult.LayerVersions[0].LayerVersionArn;
+    }
+
+    const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
+    const { sourceTemporaryDir, lambdaZipPath } =
+      await lambdaBuildDirectoryManager.init();
+
+    const nodeDependenciesFolder = join(
+      sourceTemporaryDir,
+      NODE_LAYER_SUBFOLDER,
+    );
+
+    await copyAndBuildDependencies(nodeDependenciesFolder);
+
+    await createZipFile(sourceTemporaryDir, lambdaZipPath);
+
+    const params: PublishLayerVersionCommandInput = {
+      LayerName: COMMON_LAYER_NAME,
+      Content: {
+        ZipFile: await fs.readFile(lambdaZipPath),
+      },
+      CompatibleRuntimes: [ServerlessFunctionRuntime.NODE18],
+      Description: `${version}`,
+    };
+
+    const command = new PublishLayerVersionCommand(params);
+
+    const result = await this.lambdaClient.send(command);
+
+    await lambdaBuildDirectoryManager.clean();
+
+    if (!isDefined(result.LayerVersionArn)) {
+      throw new Error('new layer version arn si undefined');
+    }
+
+    return result.LayerVersionArn;
  }

  private async checkFunctionExists(functionName: string): Promise<boolean> {
@ -95,14 +170,16 @@ export class LambdaDriver
      this.fileStorageService,
    );

+    const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
+
    const {
      sourceTemporaryDir,
      lambdaZipPath,
      javascriptFilePath,
      lambdaHandler,
-    } = await this.buildDirectoryManagerService.init();
+    } = await lambdaBuildDirectoryManager.init();

-    await fs.promises.writeFile(javascriptFilePath, javascriptCode);
+    await fs.writeFile(javascriptFilePath, javascriptCode);

    await createZipFile(sourceTemporaryDir, lambdaZipPath);

@ -111,12 +188,17 @@ export class LambdaDriver
    );

    if (!functionExists) {
+      const layerArn = await this.createLayerIfNotExists(
+        serverlessFunction.layerVersion,
+      );
+
      const params: CreateFunctionCommandInput = {
        Code: {
-          ZipFile: await fs.promises.readFile(lambdaZipPath),
+          ZipFile: await fs.readFile(lambdaZipPath),
        },
        FunctionName: serverlessFunction.id,
        Handler: lambdaHandler,
+        Layers: [layerArn],
        Role: this.lambdaRole,
        Runtime: serverlessFunction.runtime,
        Description: 'Lambda function to run user script',
@ -128,7 +210,7 @@ export class LambdaDriver
      await this.lambdaClient.send(command);
    } else {
      const params: UpdateFunctionCodeCommandInput = {
-        ZipFile: await fs.promises.readFile(lambdaZipPath),
+        ZipFile: await fs.readFile(lambdaZipPath),
        FunctionName: serverlessFunction.id,
      };

@ -137,14 +219,9 @@ export class LambdaDriver
      await this.lambdaClient.send(command);
    }

-    const waitParams = { FunctionName: serverlessFunction.id };
+    await this.waitFunctionUpdates(serverlessFunction.id, 10);

-    await waitUntilFunctionUpdatedV2(
-      { client: this.lambdaClient, maxWaitTime: 5 },
-      waitParams,
-    );
-
-    await this.buildDirectoryManagerService.clean();
+    await lambdaBuildDirectoryManager.clean();
  }

  async publish(serverlessFunction: ServerlessFunctionEntity) {
@ -167,7 +244,7 @@ export class LambdaDriver

  async execute(
    functionToExecute: ServerlessFunctionEntity,
-    payload: object | undefined = undefined,
+    payload: object,
    version: string,
  ): Promise<ServerlessExecuteResult> {
    const computedVersion =
@ -177,8 +254,11 @@ export class LambdaDriver
      computedVersion === 'draft'
        ? functionToExecute.id
        : `${functionToExecute.id}:${computedVersion}`;
+
+    await this.waitFunctionUpdates(functionToExecute.id, 10);
+
    const startTime = Date.now();
-    const params = {
+    const params: InvokeCommandInput = {
      FunctionName: functionName,
      Payload: JSON.stringify(payload),
    };
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/1/package.json
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/1/package.json
@ -0,0 +1,48 @@
+{
+  "dependencies": {
+    "@types/bcrypt": "^5.0.2",
+    "@types/deep-equal": "^1.0.4",
+    "@types/lodash.camelcase": "^4.3.9",
+    "@types/lodash.compact": "^3.0.9",
+    "@types/lodash.debounce": "^4.0.9",
+    "@types/lodash.groupby": "^4.6.9",
+    "@types/lodash.identity": "^3.0.9",
+    "@types/lodash.isempty": "^4.4.9",
+    "@types/lodash.isequal": "^4.5.8",
+    "@types/lodash.isobject": "^3.0.9",
+    "@types/lodash.kebabcase": "^4.1.9",
+    "@types/lodash.mapvalues": "^4.6.9",
+    "@types/lodash.omit": "^4.5.9",
+    "@types/lodash.pickby": "^4.6.9",
+    "@types/lodash.snakecase": "^4.1.9",
+    "@types/lodash.upperfirst": "^4.3.9",
+    "@types/uuid": "^10.0.0",
+    "archiver": "^7.0.1",
+    "axios": "^1.7.5",
+    "bcrypt": "^5.1.1",
+    "body-parser": "^1.20.2",
+    "deep-equal": "^2.2.3",
+    "jsonwebtoken": "^9.0.2",
+    "lodash.camelcase": "^4.3.0",
+    "lodash.chunk": "^4.2.0",
+    "lodash.compact": "^3.0.1",
+    "lodash.debounce": "^4.0.8",
+    "lodash.groupby": "^4.6.0",
+    "lodash.identity": "^3.0.0",
+    "lodash.isempty": "^4.4.0",
+    "lodash.isequal": "^4.5.0",
+    "lodash.isobject": "^3.0.2",
+    "lodash.kebabcase": "^4.1.1",
+    "lodash.mapvalues": "^4.6.0",
+    "lodash.merge": "^4.6.2",
+    "lodash.omit": "^4.5.0",
+    "lodash.pick": "^4.4.0",
+    "lodash.pickby": "^4.6.0",
+    "lodash.snakecase": "^4.1.1",
+    "lodash.upperfirst": "^4.3.1",
+    "nodemailer": "^6.9.14",
+    "sharp": "^0.33.5",
+    "uuid": "^10.0.0",
+    "winston": "^3.14.2"
+  }
+}
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/1/yarn.lock
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/1/yarn.lock
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/engine/.yarn/releases/yarn-4.4.0.cjs
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/engine/.yarn/releases/yarn-4.4.0.cjs
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/engine/.yarnrc.yml
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/engine/.yarnrc.yml
@ -0,0 +1,5 @@
+enableInlineHunks: true
+
+nodeLinker: node-modules
+
+yarnPath: .yarn/releases/yarn-4.4.0.cjs
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/last-layer-version.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/layers/last-layer-version.ts
@ -0,0 +1 @@
+export const LAST_LAYER_VERSION = 1;
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/local.driver.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/local.driver.ts
@ -1,6 +1,5 @@
 import { fork } from 'child_process';
-import { promises as fs } from 'fs';
-import { tmpdir } from 'os';
+import { promises as fs, existsSync } from 'fs';
 import { join } from 'path';

 import { v4 } from 'uuid';
@ -23,6 +22,9 @@ import {
  ServerlessFunctionException,
  ServerlessFunctionExceptionCode,
 } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
+import { COMMON_LAYER_NAME } from 'src/engine/integrations/serverless/drivers/constants/common-layer-name';
+import { copyAndBuildDependencies } from 'src/engine/integrations/serverless/drivers/utils/copy-and-build-dependencies';
+import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/integrations/serverless/drivers/constants/serverless-tmpdir-folder';

 export interface LocalDriverOptions {
  fileStorageService: FileStorageService;
@ -39,22 +41,40 @@ export class LocalDriver
    this.fileStorageService = options.fileStorageService;
  }

+  private getInMemoryLayerFolderPath = (version: number) => {
+    return join(SERVERLESS_TMPDIR_FOLDER, COMMON_LAYER_NAME, `${version}`);
+  };
+
+  private async createLayerIfNotExists(version: number) {
+    const inMemoryLastVersionLayerFolderPath =
+      this.getInMemoryLayerFolderPath(version);
+
+    if (existsSync(inMemoryLastVersionLayerFolderPath)) {
+      return;
+    }
+
+    await copyAndBuildDependencies(inMemoryLastVersionLayerFolderPath);
+  }
+
  async delete() {}

  async build(serverlessFunction: ServerlessFunctionEntity) {
+    await this.createLayerIfNotExists(serverlessFunction.layerVersion);
    const javascriptCode = await this.getCompiledCode(
      serverlessFunction,
      this.fileStorageService,
    );

+    const draftFolderPath = getServerlessFolder({
+      serverlessFunction,
+      version: 'draft',
+    });
+
    await this.fileStorageService.write({
      file: javascriptCode,
      name: BUILD_FILE_NAME,
      mimeType: undefined,
-      folder: getServerlessFolder({
-        serverlessFunction,
-        version: 'draft',
-      }),
+      folder: draftFolderPath,
    });
  }

@ -68,9 +88,11 @@ export class LocalDriver

  async execute(
    serverlessFunction: ServerlessFunctionEntity,
-    payload: object | undefined = undefined,
+    payload: object,
    version: string,
  ): Promise<ServerlessExecuteResult> {
+    await this.createLayerIfNotExists(serverlessFunction.layerVersion);
+
    const startTime = Date.now();
    let fileContent = '';

@ -94,7 +116,15 @@ export class LocalDriver
      throw error;
    }

-    const tmpFilePath = join(tmpdir(), `${v4()}.js`);
+    const tmpFolderPath = join(SERVERLESS_TMPDIR_FOLDER, v4());
+
+    const tmpFilePath = join(tmpFolderPath, 'index.js');
+
+    await fs.symlink(
+      this.getInMemoryLayerFolderPath(serverlessFunction.layerVersion),
+      tmpFolderPath,
+      'dir',
+    );

    const modifiedContent = `
    process.on('message', async (message) => {
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/utils/copy-and-build-dependencies.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/utils/copy-and-build-dependencies.ts
@ -0,0 +1,40 @@
+import { statSync, promises as fs } from 'fs';
+import { promisify } from 'util';
+import { exec } from 'child_process';
+import { join } from 'path';
+
+import { getLayerDependenciesDirName } from 'src/engine/integrations/serverless/drivers/utils/get-layer-dependencies-dir-name';
+
+const execPromise = promisify(exec);
+
+export const copyAndBuildDependencies = async (buildDirectory: string) => {
+  await fs.mkdir(buildDirectory, {
+    recursive: true,
+  });
+
+  await fs.cp(getLayerDependenciesDirName('latest'), buildDirectory, {
+    recursive: true,
+  });
+  await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
+    recursive: true,
+  });
+
+  try {
+    await execPromise('yarn', { cwd: buildDirectory });
+  } catch (error: any) {
+    throw new Error(error.stdout);
+  }
+  const objects = await fs.readdir(buildDirectory);
+
+  objects.forEach((object) => {
+    const fullPath = join(buildDirectory, object);
+
+    if (object === 'node_modules') return;
+
+    if (statSync(fullPath).isDirectory()) {
+      fs.rm(fullPath, { recursive: true, force: true });
+    } else {
+      fs.rm(fullPath);
+    }
+  });
+};
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/utils/get-last-layer-dependencies.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/utils/get-last-layer-dependencies.ts
@ -0,0 +1,24 @@
+import fs from 'fs/promises';
+import { join } from 'path';
+
+import { getLayerDependenciesDirName } from 'src/engine/integrations/serverless/drivers/utils/get-layer-dependencies-dir-name';
+
+export type LayerDependencies = {
+  packageJson: { dependencies: object };
+  yarnLock: string;
+};
+
+export const getLastLayerDependencies =
+  async (): Promise<LayerDependencies> => {
+    const lastVersionLayerDirName = getLayerDependenciesDirName('latest');
+    const packageJson = await fs.readFile(
+      join(lastVersionLayerDirName, 'package.json'),
+      'utf8',
+    );
+    const yarnLock = await fs.readFile(
+      join(lastVersionLayerDirName, 'yarn.lock'),
+      'utf8',
+    );
+
+    return { packageJson: JSON.parse(packageJson), yarnLock };
+  };
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/utils/get-layer-dependencies-dir-name.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/utils/get-layer-dependencies-dir-name.ts
@ -0,0 +1,12 @@
+import path from 'path';
+
+import { LAST_LAYER_VERSION } from 'src/engine/integrations/serverless/drivers/layers/last-layer-version';
+
+// Can only be used in src/engine/integrations/serverless/drivers/utils folder
+export const getLayerDependenciesDirName = (
+  version: 'latest' | 'engine' | number,
+): string => {
+  const formattedVersion = version === 'latest' ? LAST_LAYER_VERSION : version;
+
+  return path.resolve(__dirname, `../layers/${formattedVersion}`);
+};
--- a/packages/twenty-server/src/engine/integrations/serverless/drivers/services/build-directory-manager.service.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/drivers/services/build-directory-manager.service.ts
@ -1,20 +1,22 @@
-import { Injectable } from '@nestjs/common';
-
 import { join } from 'path';
-import { tmpdir } from 'os';
-import fs from 'fs';
+import * as fs from 'fs/promises';

-import fsExtra from 'fs-extra';
 import { v4 } from 'uuid';

-const TEMPORARY_LAMBDA_FOLDER = 'twenty-build-lambda-temp-folder';
+import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/integrations/serverless/drivers/constants/serverless-tmpdir-folder';
+
+export const NODE_LAYER_SUBFOLDER = 'nodejs';
+
+const TEMPORARY_LAMBDA_FOLDER = 'lambda-build';
 const TEMPORARY_LAMBDA_SOURCE_FOLDER = 'src';
 const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
 const LAMBDA_ENTRY_FILE_NAME = 'index.js';

-@Injectable()
-export class BuildDirectoryManagerService {
-  private temporaryDir = join(tmpdir(), `${TEMPORARY_LAMBDA_FOLDER}_${v4()}`);
+export class LambdaBuildDirectoryManager {
+  private temporaryDir = join(
+    SERVERLESS_TMPDIR_FOLDER,
+    `${TEMPORARY_LAMBDA_FOLDER}-${v4()}`,
+  );
  private lambdaHandler = `${LAMBDA_ENTRY_FILE_NAME.split('.')[0]}.handler`;

  async init() {
@ -25,13 +27,7 @@ export class BuildDirectoryManagerService {
    const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
    const javascriptFilePath = join(sourceTemporaryDir, LAMBDA_ENTRY_FILE_NAME);

-    if (!fs.existsSync(this.temporaryDir)) {
-      await fs.promises.mkdir(this.temporaryDir);
-      await fs.promises.mkdir(sourceTemporaryDir);
-    } else {
-      await fsExtra.emptyDir(this.temporaryDir);
-      await fs.promises.mkdir(sourceTemporaryDir);
-    }
+    await fs.mkdir(sourceTemporaryDir, { recursive: true });

    return {
      sourceTemporaryDir,
@ -42,7 +38,6 @@ export class BuildDirectoryManagerService {
  }

  async clean() {
-    await fsExtra.emptyDir(this.temporaryDir);
-    await fs.promises.rmdir(this.temporaryDir);
+    await fs.rm(this.temporaryDir, { recursive: true, force: true });
  }
 }
--- a/packages/twenty-server/src/engine/integrations/serverless/serverless-module.factory.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/serverless-module.factory.ts
@ -2,7 +2,6 @@ import { fromNodeProviderChain } from '@aws-sdk/credential-providers';

 import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
 import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
-import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
 import {
  ServerlessDriverType,
  ServerlessModuleOptions,
@ -11,7 +10,6 @@ import {
 export const serverlessModuleFactory = async (
  environmentService: EnvironmentService,
  fileStorageService: FileStorageService,
-  buildDirectoryManagerService: BuildDirectoryManagerService,
 ): Promise<ServerlessModuleOptions> => {
  const driverType = environmentService.get('SERVERLESS_TYPE');
  const options = { fileStorageService };
@ -37,7 +35,6 @@ export const serverlessModuleFactory = async (
        type: ServerlessDriverType.Lambda,
        options: {
          ...options,
-          buildDirectoryManagerService,
          credentials: accessKeyId
            ? {
                accessKeyId,
--- a/packages/twenty-server/src/engine/integrations/serverless/serverless.module.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/serverless.module.ts
@ -2,7 +2,6 @@ import { DynamicModule, Global } from '@nestjs/common';

 import { LambdaDriver } from 'src/engine/integrations/serverless/drivers/lambda.driver';
 import { LocalDriver } from 'src/engine/integrations/serverless/drivers/local.driver';
-import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
 import { SERVERLESS_DRIVER } from 'src/engine/integrations/serverless/serverless.constants';
 import {
  ServerlessDriverType,
@ -28,7 +27,7 @@ export class ServerlessModule {
    return {
      module: ServerlessModule,
      imports: options.imports || [],
-      providers: [ServerlessService, BuildDirectoryManagerService, provider],
+      providers: [ServerlessService, provider],
      exports: [ServerlessService],
    };
  }
--- a/packages/twenty-server/src/engine/integrations/serverless/serverless.service.ts
+++ b/packages/twenty-server/src/engine/integrations/serverless/serverless.service.ts
@ -29,7 +29,7 @@ export class ServerlessService implements ServerlessDriver {

  async execute(
    serverlessFunction: ServerlessFunctionEntity,
-    payload: object | undefined = undefined,
+    payload: object,
    version: string,
  ): Promise<ServerlessExecuteResult> {
    return this.driver.execute(serverlessFunction, payload, version);
--- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input.ts
@ -1,6 +1,6 @@
 import { Field, InputType } from '@nestjs/graphql';

-import { IsNotEmpty, IsObject, IsOptional, IsUUID } from 'class-validator';
+import { IsNotEmpty, IsObject, IsUUID } from 'class-validator';
 import graphqlTypeJson from 'graphql-type-json';

 import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@ -16,11 +16,9 @@ export class ExecuteServerlessFunctionInput {

  @Field(() => graphqlTypeJson, {
    description: 'Payload in JSON format',
-    nullable: true,
  })
  @IsObject()
-  @IsOptional()
-  payload?: JSON;
+  payload: JSON;

  @Field(() => String, {
    nullable: false,
--- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts
@ -43,7 +43,6 @@ export class ServerlessFunctionDTO {
  id: string;

  @IsString()
-  @IsNotEmpty()
  @Field()
  name: string;

--- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input.ts
@ -14,7 +14,6 @@ export class UpdateServerlessFunctionInput {
  id: string;

  @IsString()
-  @IsNotEmpty()
  @Field()
  name: string;

--- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts
@ -35,6 +35,9 @@ export class ServerlessFunctionEntity {
  @Column({ nullable: false, default: ServerlessFunctionRuntime.NODE18 })
  runtime: ServerlessFunctionRuntime;

+  @Column({ nullable: true })
+  layerVersion: number;
+
  @Column({
    nullable: false,
    default: ServerlessFunctionSyncStatus.NOT_READY,
--- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.resolver.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.resolver.ts
@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';

 import { FileUpload, GraphQLUpload } from 'graphql-upload';
 import { Repository } from 'typeorm';
+import graphqlTypeJson from 'graphql-type-json';

 import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
 import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
@ -51,7 +52,18 @@ export class ServerlessFunctionResolver {
    }
  }

-  @Query(() => String)
+  @Query(() => graphqlTypeJson)
+  async getAvailablePackages(@AuthWorkspace() { id: workspaceId }: Workspace) {
+    try {
+      await this.checkFeatureFlag(workspaceId);
+
+      return await this.serverlessFunctionService.getAvailablePackages();
+    } catch (error) {
+      serverlessFunctionGraphQLApiExceptionHandler(error);
+    }
+  }
+
+  @Query(() => String, { nullable: true })
  async getServerlessFunctionSourceCode(
    @Args('input') input: GetServerlessFunctionSourceCodeInput,
    @AuthWorkspace() { id: workspaceId }: Workspace,
--- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
@ -27,6 +27,8 @@ import {
 } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
 import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
 import { isDefined } from 'src/utils/is-defined';
+import { getLastLayerDependencies } from 'src/engine/integrations/serverless/drivers/utils/get-last-layer-dependencies';
+import { LAST_LAYER_VERSION } from 'src/engine/integrations/serverless/drivers/layers/last-layer-version';

@Injectable()
 export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFunctionEntity> {
@ -46,22 +48,21 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
    id: string,
    version: string,
  ) {
+    const serverlessFunction = await this.serverlessFunctionRepository.findOne({
+      where: {
+        id,
+        workspaceId,
+      },
+    });
+
+    if (!serverlessFunction) {
+      throw new ServerlessFunctionException(
+        `Function does not exist`,
+        ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
+      );
+    }
+
    try {
-      const serverlessFunction =
-        await this.serverlessFunctionRepository.findOne({
-          where: {
-            id,
-            workspaceId,
-          },
-        });
-
-      if (!serverlessFunction) {
-        throw new ServerlessFunctionException(
-          `Function does not exist`,
-          ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
-        );
-      }
-
      const folderPath = getServerlessFolder({
        serverlessFunction,
        version,
@ -75,10 +76,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
      return await readFileContent(fileStream);
    } catch (error) {
      if (error.code === FileStorageExceptionCode.FILE_NOT_FOUND) {
-        throw new ServerlessFunctionException(
-          `Function Version '${version}' does not exist`,
-          ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
-        );
+        return;
      }
      throw error;
    }
@ -87,7 +85,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
  async executeOneServerlessFunction(
    id: string,
    workspaceId: string,
-    payload: object | undefined = undefined,
+    payload: object,
    version = 'latest',
  ): Promise<ServerlessExecuteResult> {
    await this.throttleExecution(workspaceId);
@ -106,15 +104,6 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
      );
    }

-    if (
-      functionToExecute.syncStatus === ServerlessFunctionSyncStatus.NOT_READY
-    ) {
-      await this.serverlessService.build(functionToExecute, version);
-      await super.updateOne(functionToExecute.id, {
-        syncStatus: ServerlessFunctionSyncStatus.READY,
-      });
-    }
-
    return this.serverlessService.execute(functionToExecute, payload, version);
  }

@ -144,8 +133,8 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
      );

      if (
-        serverlessFunctionCreateHash(latestCode) ===
-        serverlessFunctionCreateHash(draftCode)
+        serverlessFunctionCreateHash(latestCode || '') ===
+        serverlessFunctionCreateHash(draftCode || '')
      ) {
        throw new Error(
          'Cannot publish a new version when code has not changed',
@ -224,6 +213,9 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
      name: serverlessFunctionInput.name,
      description: serverlessFunctionInput.description,
      syncStatus: ServerlessFunctionSyncStatus.NOT_READY,
+      sourceCodeHash: serverlessFunctionCreateHash(
+        serverlessFunctionInput.code,
+      ),
    });

    const fileFolder = getServerlessFolder({
@ -238,9 +230,34 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
      folder: fileFolder,
    });

+    await this.serverlessService.build(existingServerlessFunction, 'draft');
+    await super.updateOne(existingServerlessFunction.id, {
+      syncStatus: ServerlessFunctionSyncStatus.READY,
+    });
+
    return await this.findById(existingServerlessFunction.id);
  }

+  async getAvailablePackages() {
+    const { packageJson, yarnLock } = await getLastLayerDependencies();
+
+    const packageVersionRegex = /^"([^@]+)@.*?":\n\s+version: (.+)$/gm;
+    const versions: Record<string, string> = {};
+
+    let match: RegExpExecArray | null;
+
+    while ((match = packageVersionRegex.exec(yarnLock)) !== null) {
+      const packageName = match[1].split('@', 1)[0];
+      const version = match[2];
+
+      if (packageJson.dependencies[packageName]) {
+        versions[packageName] = version;
+      }
+    }
+
+    return versions;
+  }
+
  async createOneServerlessFunction(
    serverlessFunctionInput: CreateServerlessFunctionFromFileInput,
    code: FileUpload | string,
@ -258,6 +275,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
      ...serverlessFunctionInput,
      workspaceId,
      sourceCodeHash: serverlessFunctionCreateHash(typescriptCode),
+      layerVersion: LAST_LAYER_VERSION,
    });

    const draftFileFolder = getServerlessFolder({
@ -272,6 +290,8 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
      folder: draftFileFolder,
    });

+    await this.serverlessService.build(createdServerlessFunction, 'draft');
+
    return await this.findById(createdServerlessFunction.id);
  }
				`@ -0,0 +1 @@`
				`export const COMMON_LAYER_NAME = 'common-layer';`