6181 workflows create a custom code executor (#6235)

Closes #6181

## Testing
- download Altair graphql dev tool https://altairgraphql.dev/#download
- create a file locally `test.ts` containing:
```
export const handler = async (event: object, context: object) => {
  return { test: 'toto', data: event['data'] };
}
```
- play those requests in Altair:
mutation UpsertFunction($file: Upload!) {
  upsertFunction(name: "toto", file: $file)
}

mutation ExecFunction {
  executeFunction(name:"toto", payload: {data: "titi"})
}
- it will run the local driver, add those env variable to test with
lambda driver
```
CUSTOM_CODE_ENGINE_DRIVER_TYPE=lambda
LAMBDA_REGION=eu-west-2
LAMBDA_ROLE=<ASK_ME>
```

packages/twenty-server/src/engine/integrations/serverless/drivers/base-serverless.driver.ts

@@ -0,0 +1,33 @@
+import { join } from 'path';
+
+import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
+
+import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
+import { SOURCE_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/source-file-name';
+import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
+import { compileTypescript } from 'src/engine/integrations/serverless/drivers/utils/compile-typescript';
+import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
+
+export class BaseServerlessDriver {
+  getFolderPath(serverlessFunction: ServerlessFunctionEntity) {
+    return join(
+      FileFolder.ServerlessFunction,
+      serverlessFunction.workspaceId,
+      serverlessFunction.id,
+    );
+  }
+
+  async getCompiledCode(
+    serverlessFunction: ServerlessFunctionEntity,
+    fileStorageService: FileStorageService,
+  ) {
+    const folderPath = this.getFolderPath(serverlessFunction);
+    const fileStream = await fileStorageService.read({
+      folderPath,
+      filename: SOURCE_FILE_NAME,
+    });
+    const typescriptCode = await readFileContent(fileStream);
+
+    return compileTypescript(typescriptCode);
+  }
+}

packages/twenty-server/src/engine/integrations/serverless/drivers/constants/build-file-name.ts

@@ -0,0 +1 @@
+export const BUILD_FILE_NAME = 'build.js';

packages/twenty-server/src/engine/integrations/serverless/drivers/constants/source-file-name.ts

@@ -0,0 +1 @@
+export const SOURCE_FILE_NAME = 'source.ts';

packages/twenty-server/src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface.ts

@@ -0,0 +1,9 @@
+import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
+
+export interface ServerlessDriver {
+  build(serverlessFunction: ServerlessFunctionEntity): Promise<void>;
+  execute(
+    serverlessFunction: ServerlessFunctionEntity,
+    payload: object | undefined,
+  ): Promise<object>;
+}

packages/twenty-server/src/engine/integrations/serverless/drivers/lambda.driver.ts

@@ -0,0 +1,100 @@
+import fs from 'fs';
+
+import {
+  CreateFunctionCommand,
+  Lambda,
+  LambdaClientConfig,
+  InvokeCommand,
+} from '@aws-sdk/client-lambda';
+import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
+
+import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
+
+import { createZipFile } from 'src/engine/integrations/serverless/drivers/utils/create-zip-file';
+import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
+import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
+import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
+import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
+
+export interface LambdaDriverOptions extends LambdaClientConfig {
+  fileStorageService: FileStorageService;
+  buildDirectoryManagerService: BuildDirectoryManagerService;
+  region: string;
+  role: string;
+}
+
+export class LambdaDriver
+  extends BaseServerlessDriver
+  implements ServerlessDriver
+{
+  private readonly lambdaClient: Lambda;
+  private readonly lambdaRole: string;
+  private readonly fileStorageService: FileStorageService;
+  private readonly buildDirectoryManagerService: BuildDirectoryManagerService;
+
+  constructor(options: LambdaDriverOptions) {
+    super();
+    const { region, role, ...lambdaOptions } = options;
+
+    this.lambdaClient = new Lambda({ ...lambdaOptions, region });
+    this.lambdaRole = role;
+    this.fileStorageService = options.fileStorageService;
+    this.buildDirectoryManagerService = options.buildDirectoryManagerService;
+  }
+
+  async build(serverlessFunction: ServerlessFunctionEntity) {
+    const javascriptCode = await this.getCompiledCode(
+      serverlessFunction,
+      this.fileStorageService,
+    );
+
+    const {
+      sourceTemporaryDir,
+      lambdaZipPath,
+      javascriptFilePath,
+      lambdaHandler,
+    } = await this.buildDirectoryManagerService.init();
+
+    await fs.promises.writeFile(javascriptFilePath, javascriptCode);
+
+    await createZipFile(sourceTemporaryDir, lambdaZipPath);
+
+    const params: CreateFunctionCommandInput = {
+      Code: {
+        ZipFile: await fs.promises.readFile(lambdaZipPath),
+      },
+      FunctionName: serverlessFunction.id,
+      Handler: lambdaHandler,
+      Role: this.lambdaRole,
+      Runtime: 'nodejs18.x',
+      Description: 'Lambda function to run user script',
+      Timeout: 900,
+    };
+
+    const command = new CreateFunctionCommand(params);
+
+    await this.lambdaClient.send(command);
+
+    await this.buildDirectoryManagerService.clean();
+  }
+
+  async execute(
+    functionToExecute: ServerlessFunctionEntity,
+    payload: object | undefined = undefined,
+  ): Promise<object> {
+    const params = {
+      FunctionName: functionToExecute.id,
+      Payload: JSON.stringify(payload),
+    };
+
+    const command = new InvokeCommand(params);
+
+    const result = await this.lambdaClient.send(command);
+
+    if (!result.Payload) {
+      return {};
+    }
+
+    return JSON.parse(result.Payload.transformToString());
+  }
+}

packages/twenty-server/src/engine/integrations/serverless/drivers/local.driver.ts

@@ -0,0 +1,94 @@
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { promises as fs } from 'fs';
+import { fork } from 'child_process';
+
+import { v4 } from 'uuid';
+
+import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
+
+import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
+import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
+import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
+import { BUILD_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/build-file-name';
+import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
+
+export interface LocalDriverOptions {
+  fileStorageService: FileStorageService;
+}
+
+export class LocalDriver
+  extends BaseServerlessDriver
+  implements ServerlessDriver
+{
+  private readonly fileStorageService: FileStorageService;
+
+  constructor(options: LocalDriverOptions) {
+    super();
+    this.fileStorageService = options.fileStorageService;
+  }
+
+  async build(serverlessFunction: ServerlessFunctionEntity) {
+    const javascriptCode = await this.getCompiledCode(
+      serverlessFunction,
+      this.fileStorageService,
+    );
+
+    await this.fileStorageService.write({
+      file: javascriptCode,
+      name: BUILD_FILE_NAME,
+      mimeType: undefined,
+      folder: this.getFolderPath(serverlessFunction),
+    });
+  }
+
+  async execute(
+    serverlessFunction: ServerlessFunctionEntity,
+    payload: object | undefined = undefined,
+  ): Promise<object> {
+    const fileStream = await this.fileStorageService.read({
+      folderPath: this.getFolderPath(serverlessFunction),
+      filename: BUILD_FILE_NAME,
+    });
+    const fileContent = await readFileContent(fileStream);
+
+    const tmpFilePath = join(tmpdir(), `${v4()}.js`);
+
+    const modifiedContent = `
+    process.on('message', async (message) => {
+      const { event, context } = message;
+      const result = await handler(event, context);
+      process.send(result);
+    });
+
+    ${fileContent}
+    `;
+
+    await fs.writeFile(tmpFilePath, modifiedContent);
+
+    return await new Promise((resolve, reject) => {
+      const child = fork(tmpFilePath);
+
+      child.on('message', (message: object) => {
+        resolve(message);
+        child.kill();
+        fs.unlink(tmpFilePath);
+      });
+
+      child.on('error', (error) => {
+        reject(error);
+        child.kill();
+        fs.unlink(tmpFilePath);
+      });
+
+      child.on('exit', (code) => {
+        if (code && code !== 0) {
+          reject(new Error(`Child process exited with code ${code}`));
+          fs.unlink(tmpFilePath);
+        }
+      });
+
+      child.send({ event: payload });
+    });
+  }
+}

packages/twenty-server/src/engine/integrations/serverless/drivers/services/build-directory-manager.service.ts

@@ -0,0 +1,48 @@
+import { Injectable } from '@nestjs/common';
+
+import { join } from 'path';
+import { tmpdir } from 'os';
+import fs from 'fs';
+
+import fsExtra from 'fs-extra';
+import { v4 } from 'uuid';
+
+const TEMPORARY_LAMBDA_FOLDER = 'twenty-build-lambda-temp-folder';
+const TEMPORARY_LAMBDA_SOURCE_FOLDER = 'src';
+const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
+const LAMBDA_ENTRY_FILE_NAME = 'index.js';
+
+@Injectable()
+export class BuildDirectoryManagerService {
+  private temporaryDir = join(tmpdir(), `${TEMPORARY_LAMBDA_FOLDER}_${v4()}`);
+  private lambdaHandler = `${LAMBDA_ENTRY_FILE_NAME.split('.')[0]}.handler`;
+
+  async init() {
+    const sourceTemporaryDir = join(
+      this.temporaryDir,
+      TEMPORARY_LAMBDA_SOURCE_FOLDER,
+    );
+    const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
+    const javascriptFilePath = join(sourceTemporaryDir, LAMBDA_ENTRY_FILE_NAME);
+
+    if (!fs.existsSync(this.temporaryDir)) {
+      await fs.promises.mkdir(this.temporaryDir);
+      await fs.promises.mkdir(sourceTemporaryDir);
+    } else {
+      await fsExtra.emptyDir(this.temporaryDir);
+      await fs.promises.mkdir(sourceTemporaryDir);
+    }
+
+    return {
+      sourceTemporaryDir,
+      lambdaZipPath,
+      javascriptFilePath,
+      lambdaHandler: this.lambdaHandler,
+    };
+  }
+
+  async clean() {
+    await fsExtra.emptyDir(this.temporaryDir);
+    await fs.promises.rmdir(this.temporaryDir);
+  }
+}

packages/twenty-server/src/engine/integrations/serverless/drivers/utils/compile-typescript.ts

@@ -0,0 +1,19 @@
+import ts from 'typescript';
+
+export const compileTypescript = (typescriptCode: string): string => {
+  const options: ts.CompilerOptions = {
+    module: ts.ModuleKind.CommonJS,
+    target: ts.ScriptTarget.ES2017,
+    moduleResolution: ts.ModuleResolutionKind.Node10,
+    esModuleInterop: true,
+    resolveJsonModule: true,
+    allowSyntheticDefaultImports: true,
+    types: ['node'],
+  };
+
+  const result = ts.transpileModule(typescriptCode, {
+    compilerOptions: options,
+  });
+
+  return result.outputText;
+};

packages/twenty-server/src/engine/integrations/serverless/drivers/utils/create-zip-file.ts

@@ -0,0 +1,21 @@
+import fs from 'fs';
+import { pipeline } from 'stream/promises';
+
+import archiver from 'archiver';
+
+export const createZipFile = async (
+  sourceDir: string,
+  outPath: string,
+): Promise<void> => {
+  const output = fs.createWriteStream(outPath);
+  const archive = archiver('zip', {
+    zlib: { level: 9 }, // Compression level
+  });
+
+  const p = pipeline(archive, output);
+
+  archive.directory(sourceDir, false);
+  archive.finalize();
+
+  return p;
+};