8725 workflow avoid serverless function autosave errors (#8916)

See issue #8725 
- Build function asynchronously using a job
- prevent useless builds
- run promises simultaneously

Todo:
- fix outputSchema computing

packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts

@@ -2,6 +2,7 @@ import { OnModuleDestroy } from '@nestjs/common';
 
 import { JobsOptions, Queue, QueueOptions, Worker } from 'bullmq';
 import omitBy from 'lodash.omitby';
+import { v4 } from 'uuid';
 
 import {
   QueueCronJobOptions,
@@ -15,6 +16,8 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
 
 export type BullMQDriverOptions = QueueOptions;
 
+const V4_LENGTH = 36;
+
 export class BullMQDriver implements MessageQueueDriver, OnModuleDestroy {
   private queueMap: Record<MessageQueue, Queue> = {} as Record<
     MessageQueue,
@@ -107,8 +110,22 @@ export class BullMQDriver implements MessageQueueDriver, OnModuleDestroy {
         `Queue ${queueName} is not registered, make sure you have added it as a queue provider`,
       );
     }
+
+    // This ensures only one waiting job can be queued for a specific option.id
+    if (options?.id) {
+      const waitingJobs = await this.queueMap[queueName].getJobs(['waiting']);
+
+      const isJobAlreadyWaiting = waitingJobs.some(
+        (job) => job.id?.slice(0, -(V4_LENGTH + 1)) === options.id,
+      );
+
+      if (isJobAlreadyWaiting) {
+        return;
+      }
+    }
+
     const queueOptions: JobsOptions = {
-      jobId: options?.id,
+      jobId: options?.id ? `${options.id}-${v4()}` : undefined, // We add V4() to id to make sure ids are uniques so we can add a waiting job when a job related with the same option.id is running
       priority: options?.priority,
       attempts: 1 + (options?.retryLimit || 0),
       removeOnComplete: 100,

packages/twenty-server/src/engine/core-modules/message-queue/drivers/pg-boss.driver.ts

@@ -98,6 +98,7 @@ export class PgBossDriver
         ? {
             ...options,
             singletonKey: options?.id,
+            useSingletonQueue: true, // When used with singletonKey, ensures only one job can be queued. See https://logsnag.com/blog/deep-dive-into-background-jobs-with-pg-boss-and-typescript
           }
         : {},
     );

packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts

@@ -54,6 +54,8 @@ import { compileTypescript } from 'src/engine/core-modules/serverless/drivers/ut
 import { ENV_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/env-file-name';
 import { OUTDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/outdir-folder';
 
+const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 30;
+
 export interface LambdaDriverOptions extends LambdaClientConfig {
   fileStorageService: FileStorageService;
   region: string;
@@ -75,7 +77,7 @@ export class LambdaDriver implements ServerlessDriver {
 
   private async waitFunctionUpdates(
     serverlessFunctionId: string,
-    maxWaitTime: number,
+    maxWaitTime: number = UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
   ) {
     const waitParams = { FunctionName: serverlessFunctionId };
 
@@ -263,12 +265,12 @@ export class LambdaDriver implements ServerlessDriver {
         updateConfigurationParams,
       );
 
-      await this.waitFunctionUpdates(serverlessFunction.id, 10);
+      await this.waitFunctionUpdates(serverlessFunction.id);
 
       await this.lambdaClient.send(updateConfigurationCommand);
     }
 
-    await this.waitFunctionUpdates(serverlessFunction.id, 10);
+    await this.waitFunctionUpdates(serverlessFunction.id);
   }
 
   async publish(serverlessFunction: ServerlessFunctionEntity) {
@@ -316,7 +318,9 @@ export class LambdaDriver implements ServerlessDriver {
         ? functionToExecute.id
         : `${functionToExecute.id}:${computedVersion}`;
 
-    await this.waitFunctionUpdates(functionToExecute.id, 10);
+    if (version === 'draft') {
+      await this.waitFunctionUpdates(functionToExecute.id);
+    }
 
     const startTime = Date.now();
 

packages/twenty-server/src/engine/core-modules/serverless/drivers/utils/get-last-layer-dependencies.ts

@@ -12,14 +12,10 @@ export const getLayerDependencies = async (
   layerVersion: number | 'latest',
 ): Promise<LayerDependencies> => {
   const lastVersionLayerDirName = getLayerDependenciesDirName(layerVersion);
-  const packageJson = await fs.readFile(
-    join(lastVersionLayerDirName, 'package.json'),
-    'utf8',
-  );
-  const yarnLock = await fs.readFile(
-    join(lastVersionLayerDirName, 'yarn.lock'),
-    'utf8',
-  );
+  const [packageJson, yarnLock] = await Promise.all([
+    fs.readFile(join(lastVersionLayerDirName, 'package.json'), 'utf8'),
+    fs.readFile(join(lastVersionLayerDirName, 'yarn.lock'), 'utf8'),
+  ]);
 
   return { packageJson: JSON.parse(packageJson), yarnLock };
 };