22 branches 3 (#13181)

This PR does not produce any functional changes for our users. It
prepares the branches for workflows by:

- decommissioning `output` and `context` fields or `workflowRun` records
and use newly created `state` field from front-end and back-end
- use `stepStatus` computed by `back-end` in `front-end`
- add utils and types in `twenty-shared/workflow` (not completed, a
follow-up is scheduled
https://github.com/twentyhq/core-team-issues/issues/1211)
- add concurrency to `workflowQueue` message queue to avoid weird branch
execution when using forms in workflow branches
- add a WithLock decorator for better dev experience of
`CacheLockService.withLock` usage

Here is an example of such a workflow running (front branch display is
not yet done that's why it looks ugly) ->
https://discord.com/channels/1130383047699738754/1258024460238192691/1392897615171158098

packages/twenty-server/src/engine/core-modules/cache-lock/with-lock.decorator.ts

@@ -0,0 +1,48 @@
+import { Inject } from '@nestjs/common';
+
+import {
+  CacheLockOptions,
+  CacheLockService,
+} from 'src/engine/core-modules/cache-lock/cache-lock.service';
+
+export const WithLock = (
+  lockKeyParamPath: string,
+  options?: CacheLockOptions,
+): MethodDecorator => {
+  const injectCacheLockService = Inject(CacheLockService);
+
+  return function (target, propertyKey, descriptor: PropertyDescriptor) {
+    injectCacheLockService(target, 'cacheLockService');
+
+    const originalMethod = descriptor.value;
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    descriptor.value = async function (...args: any[]) {
+      const self = this as { cacheLockService: CacheLockService };
+
+      if (!self.cacheLockService) {
+        throw new Error('cacheLockService not available on instance');
+      }
+
+      if (typeof args[0] !== 'object') {
+        throw new Error(
+          `You must use one object parameter to use @WithLock decorator. Received ${args}`,
+        );
+      }
+
+      const key = args[0][lockKeyParamPath];
+
+      if (typeof key !== 'string') {
+        throw new Error(
+          `Could not resolve lock key from path "${lockKeyParamPath}" on first argument`,
+        );
+      }
+
+      return await self.cacheLockService.withLock(
+        () => originalMethod.apply(self, args),
+        key,
+        options,
+      );
+    };
+  };
+};

packages/twenty-server/src/engine/core-modules/message-queue/decorators/process.decorator.ts

@@ -1,21 +1,11 @@
 import { SetMetadata } from '@nestjs/common';
-import { isString } from '@nestjs/common/utils/shared.utils';
 
 import { PROCESS_METADATA } from 'src/engine/core-modules/message-queue/message-queue.constants';
 
 export interface MessageQueueProcessOptions {
   jobName: string;
-  concurrency?: number;
 }
 
-export function Process(jobName: string): MethodDecorator;
-export function Process(options: MessageQueueProcessOptions): MethodDecorator;
-export function Process(
-  nameOrOptions: string | MessageQueueProcessOptions,
-): MethodDecorator {
-  const options = isString(nameOrOptions)
-    ? { jobName: nameOrOptions }
-    : nameOrOptions;
-
-  return SetMetadata(PROCESS_METADATA, options || {});
+export function Process(jobName: string): MethodDecorator {
+  return SetMetadata(PROCESS_METADATA, { jobName });
 }

packages/twenty-server/src/engine/core-modules/message-queue/decorators/processor.decorator.ts

@@ -1,12 +1,9 @@
 import { Scope, SetMetadata } from '@nestjs/common';
 import { SCOPE_OPTIONS_METADATA } from '@nestjs/common/constants';
 
-import { MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';
-
 import {
   MessageQueue,
   PROCESSOR_METADATA,
-  WORKER_METADATA,
 } from 'src/engine/core-modules/message-queue/message-queue.constants';
 
 export interface MessageQueueProcessorOptions {
@@ -24,16 +21,7 @@ export interface MessageQueueProcessorOptions {
  * Represents a worker that is able to process jobs from the queue.
  * @param queueName name of the queue to process
  */
-export function Processor(queueName: string): ClassDecorator;
-/**
- * Represents a worker that is able to process jobs from the queue.
- * @param queueName name of the queue to process
- * @param workerOptions additional worker options
- */
-export function Processor(
-  queueName: string,
-  workerOptions: MessageQueueWorkerOptions,
-): ClassDecorator;
+export function Processor(queueName: MessageQueue): ClassDecorator;
 /**
  * Represents a worker that is able to process jobs from the queue.
  * @param processorOptions processor options
@@ -41,21 +29,11 @@ export function Processor(
 export function Processor(
   processorOptions: MessageQueueProcessorOptions,
 ): ClassDecorator;
-/**
- * Represents a worker that is able to process jobs from the queue.
- * @param processorOptions processor options (Nest-specific)
- * @param workerOptions additional Bull worker options
- */
 export function Processor(
-  processorOptions: MessageQueueProcessorOptions,
-  workerOptions: MessageQueueWorkerOptions,
-): ClassDecorator;
-export function Processor(
-  queueNameOrOptions?: string | MessageQueueProcessorOptions,
-  maybeWorkerOptions?: MessageQueueWorkerOptions,
+  queueNameOrOptions: string | MessageQueueProcessorOptions,
 ): ClassDecorator {
   const options =
-    queueNameOrOptions && typeof queueNameOrOptions === 'object'
+    typeof queueNameOrOptions === 'object'
       ? queueNameOrOptions
       : { queueName: queueNameOrOptions };
 
@@ -63,7 +41,5 @@ export function Processor(
   return (target: Function) => {
     SetMetadata(SCOPE_OPTIONS_METADATA, options)(target);
     SetMetadata(PROCESSOR_METADATA, options)(target);
-    maybeWorkerOptions &&
-      SetMetadata(WORKER_METADATA, maybeWorkerOptions)(target);
   };
 }

packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts

@@ -1,6 +1,5 @@
 export const PROCESSOR_METADATA = Symbol('message-queue:processor_metadata');
 export const PROCESS_METADATA = Symbol('message-queue:process_metadata');
-export const WORKER_METADATA = Symbol('bullmq:worker_metadata');
 export const QUEUE_DRIVER = Symbol('message-queue:queue_driver');
 
 export enum MessageQueue {
@@ -13,11 +12,8 @@ export enum MessageQueue {
   contactCreationQueue = 'contact-creation-queue',
   billingQueue = 'billing-queue',
   workspaceQueue = 'workspace-queue',
-  recordPositionBackfillQueue = 'record-position-backfill-queue',
   entityEventsToDbQueue = 'entity-events-to-db-queue',
-  testQueue = 'test-queue',
   workflowQueue = 'workflow-queue',
-  serverlessFunctionQueue = 'serverless-function-queue',
   deleteCascadeQueue = 'delete-cascade-queue',
   subscriptionsQueue = 'subscriptions-queue',
 }