Serverless function improvements (#6769)

- add layer for lambda execution
- add layer for local execution
- add package resolve for the monaco editor
- add route to get installed package for serverless functions
- add layer versioning

packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input.ts

@@ -1,6 +1,6 @@
 import { Field, InputType } from '@nestjs/graphql';
 
-import { IsNotEmpty, IsObject, IsOptional, IsUUID } from 'class-validator';
+import { IsNotEmpty, IsObject, IsUUID } from 'class-validator';
 import graphqlTypeJson from 'graphql-type-json';
 
 import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@@ -16,11 +16,9 @@ export class ExecuteServerlessFunctionInput {
 
   @Field(() => graphqlTypeJson, {
     description: 'Payload in JSON format',
-    nullable: true,
   })
   @IsObject()
-  @IsOptional()
-  payload?: JSON;
+  payload: JSON;
 
   @Field(() => String, {
     nullable: false,

packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts

@@ -43,7 +43,6 @@ export class ServerlessFunctionDTO {
   id: string;
 
   @IsString()
-  @IsNotEmpty()
   @Field()
   name: string;
 

packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input.ts

@@ -14,7 +14,6 @@ export class UpdateServerlessFunctionInput {
   id: string;
 
   @IsString()
-  @IsNotEmpty()
   @Field()
   name: string;
 

packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts

@@ -35,6 +35,9 @@ export class ServerlessFunctionEntity {
   @Column({ nullable: false, default: ServerlessFunctionRuntime.NODE18 })
   runtime: ServerlessFunctionRuntime;
 
+  @Column({ nullable: true })
+  layerVersion: number;
+
   @Column({
     nullable: false,
     default: ServerlessFunctionSyncStatus.NOT_READY,

packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.resolver.ts

@@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 
 import { FileUpload, GraphQLUpload } from 'graphql-upload';
 import { Repository } from 'typeorm';
+import graphqlTypeJson from 'graphql-type-json';
 
 import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
 import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
@@ -51,7 +52,18 @@ export class ServerlessFunctionResolver {
     }
   }
 
-  @Query(() => String)
+  @Query(() => graphqlTypeJson)
+  async getAvailablePackages(@AuthWorkspace() { id: workspaceId }: Workspace) {
+    try {
+      await this.checkFeatureFlag(workspaceId);
+
+      return await this.serverlessFunctionService.getAvailablePackages();
+    } catch (error) {
+      serverlessFunctionGraphQLApiExceptionHandler(error);
+    }
+  }
+
+  @Query(() => String, { nullable: true })
   async getServerlessFunctionSourceCode(
     @Args('input') input: GetServerlessFunctionSourceCodeInput,
     @AuthWorkspace() { id: workspaceId }: Workspace,

packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts

@@ -27,6 +27,8 @@ import {
 } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
 import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
 import { isDefined } from 'src/utils/is-defined';
+import { getLastLayerDependencies } from 'src/engine/integrations/serverless/drivers/utils/get-last-layer-dependencies';
+import { LAST_LAYER_VERSION } from 'src/engine/integrations/serverless/drivers/layers/last-layer-version';
 
 @Injectable()
 export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFunctionEntity> {
@@ -46,22 +48,21 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
     id: string,
     version: string,
   ) {
+    const serverlessFunction = await this.serverlessFunctionRepository.findOne({
+      where: {
+        id,
+        workspaceId,
+      },
+    });
+
+    if (!serverlessFunction) {
+      throw new ServerlessFunctionException(
+        `Function does not exist`,
+        ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
+      );
+    }
+
     try {
-      const serverlessFunction =
-        await this.serverlessFunctionRepository.findOne({
-          where: {
-            id,
-            workspaceId,
-          },
-        });
-
-      if (!serverlessFunction) {
-        throw new ServerlessFunctionException(
-          `Function does not exist`,
-          ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
-        );
-      }
-
       const folderPath = getServerlessFolder({
         serverlessFunction,
         version,
@@ -75,10 +76,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
       return await readFileContent(fileStream);
     } catch (error) {
       if (error.code === FileStorageExceptionCode.FILE_NOT_FOUND) {
-        throw new ServerlessFunctionException(
-          `Function Version '${version}' does not exist`,
-          ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
-        );
+        return;
       }
       throw error;
     }
@@ -87,7 +85,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
   async executeOneServerlessFunction(
     id: string,
     workspaceId: string,
-    payload: object | undefined = undefined,
+    payload: object,
     version = 'latest',
   ): Promise<ServerlessExecuteResult> {
     await this.throttleExecution(workspaceId);
@@ -106,15 +104,6 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
       );
     }
 
-    if (
-      functionToExecute.syncStatus === ServerlessFunctionSyncStatus.NOT_READY
-    ) {
-      await this.serverlessService.build(functionToExecute, version);
-      await super.updateOne(functionToExecute.id, {
-        syncStatus: ServerlessFunctionSyncStatus.READY,
-      });
-    }
-
     return this.serverlessService.execute(functionToExecute, payload, version);
   }
 
@@ -144,8 +133,8 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
       );
 
       if (
-        serverlessFunctionCreateHash(latestCode) ===
-        serverlessFunctionCreateHash(draftCode)
+        serverlessFunctionCreateHash(latestCode || '') ===
+        serverlessFunctionCreateHash(draftCode || '')
       ) {
         throw new Error(
           'Cannot publish a new version when code has not changed',
@@ -224,6 +213,9 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
       name: serverlessFunctionInput.name,
       description: serverlessFunctionInput.description,
       syncStatus: ServerlessFunctionSyncStatus.NOT_READY,
+      sourceCodeHash: serverlessFunctionCreateHash(
+        serverlessFunctionInput.code,
+      ),
     });
 
     const fileFolder = getServerlessFolder({
@@ -238,9 +230,34 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
       folder: fileFolder,
     });
 
+    await this.serverlessService.build(existingServerlessFunction, 'draft');
+    await super.updateOne(existingServerlessFunction.id, {
+      syncStatus: ServerlessFunctionSyncStatus.READY,
+    });
+
     return await this.findById(existingServerlessFunction.id);
   }
 
+  async getAvailablePackages() {
+    const { packageJson, yarnLock } = await getLastLayerDependencies();
+
+    const packageVersionRegex = /^"([^@]+)@.*?":\n\s+version: (.+)$/gm;
+    const versions: Record<string, string> = {};
+
+    let match: RegExpExecArray | null;
+
+    while ((match = packageVersionRegex.exec(yarnLock)) !== null) {
+      const packageName = match[1].split('@', 1)[0];
+      const version = match[2];
+
+      if (packageJson.dependencies[packageName]) {
+        versions[packageName] = version;
+      }
+    }
+
+    return versions;
+  }
+
   async createOneServerlessFunction(
     serverlessFunctionInput: CreateServerlessFunctionFromFileInput,
     code: FileUpload | string,
@@ -258,6 +275,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
       ...serverlessFunctionInput,
       workspaceId,
       sourceCodeHash: serverlessFunctionCreateHash(typescriptCode),
+      layerVersion: LAST_LAYER_VERSION,
     });
 
     const draftFileFolder = getServerlessFolder({
@@ -272,6 +290,8 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
       folder: draftFileFolder,
     });
 
+    await this.serverlessService.build(createdServerlessFunction, 'draft');
+
     return await this.findById(createdServerlessFunction.id);
   }