Serverless function timeout concerns (#9689)

closes https://github.com/twentyhq/core-team-issues/issues/242
- unify timeout behavior between local and lambda
- add timeout in serverless entity
- set timeout default to 300s (5min)

packages/twenty-front/nyc.config.cjs

@@ -9,8 +9,8 @@ const globalCoverage = {
 
 const modulesCoverage = {
   branches: 25,
-  statements: 49,
+  statements: 44,
-  lines: 50,
+  lines: 45,
   functions: 38,
   include: ['src/modules/**/*'],
   exclude: ['src/**/*.ts'],

packages/twenty-front/src/generated-metadata/gql.ts

@@ -33,7 +33,7 @@ const documents = {
     "\n  mutation DeleteOneFieldMetadataItem($idToDelete: UUID!) {\n    deleteOneField(input: { id: $idToDelete }) {\n      id\n      type\n      name\n      label\n      description\n      icon\n      isCustom\n      isActive\n      isNullable\n      createdAt\n      updatedAt\n      settings\n    }\n  }\n": types.DeleteOneFieldMetadataItemDocument,
     "\n  mutation DeleteOneRelationMetadataItem($idToDelete: UUID!) {\n    deleteOneRelation(input: { id: $idToDelete }) {\n      id\n    }\n  }\n": types.DeleteOneRelationMetadataItemDocument,
     "\n  query ObjectMetadataItems(\n    $objectFilter: objectFilter\n    $fieldFilter: fieldFilter\n  ) {\n    objects(paging: { first: 1000 }, filter: $objectFilter) {\n      edges {\n        node {\n          id\n          dataSourceId\n          nameSingular\n          namePlural\n          labelSingular\n          labelPlural\n          description\n          icon\n          isCustom\n          isRemote\n          isActive\n          isSystem\n          createdAt\n          updatedAt\n          labelIdentifierFieldMetadataId\n          imageIdentifierFieldMetadataId\n          shortcut\n          isLabelSyncedWithName\n          indexMetadatas(paging: { first: 100 }) {\n            edges {\n              node {\n                id\n                createdAt\n                updatedAt\n                name\n                indexWhereClause\n                indexType\n                isUnique\n                indexFieldMetadatas(paging: { first: 100 }) {\n                  edges {\n                    node {\n                      id\n                      createdAt\n                      updatedAt\n                      order\n                      fieldMetadataId\n                    }\n                  }\n                }\n              }\n            }\n          }\n          fields(paging: { first: 1000 }, filter: $fieldFilter) {\n            edges {\n              node {\n                id\n                type\n                name\n                label\n                description\n                icon\n                isCustom\n                isActive\n                isSystem\n                isNullable\n                isUnique\n                createdAt\n                updatedAt\n                defaultValue\n                options\n                settings\n                isLabelSyncedWithName\n                relationDefinition {\n                  relationId\n                  direction\n                  sourceObjectMetadata {\n                    id\n                    nameSingular\n                    namePlural\n                  }\n                  sourceFieldMetadata {\n                    id\n                    name\n                  }\n                  targetObjectMetadata {\n                    id\n                    nameSingular\n                    namePlural\n                  }\n                  targetFieldMetadata {\n                    id\n                    name\n                  }\n                }\n              }\n            }\n            pageInfo {\n              hasNextPage\n              hasPreviousPage\n              startCursor\n              endCursor\n            }\n          }\n        }\n      }\n      pageInfo {\n        hasNextPage\n        hasPreviousPage\n        startCursor\n        endCursor\n      }\n    }\n  }\n": types.ObjectMetadataItemsDocument,
-    "\n  fragment ServerlessFunctionFields on ServerlessFunction {\n    id\n    name\n    description\n    runtime\n    syncStatus\n    latestVersion\n    latestVersionInputSchema\n    publishedVersions\n    createdAt\n    updatedAt\n  }\n": types.ServerlessFunctionFieldsFragmentDoc,
+    "\n  fragment ServerlessFunctionFields on ServerlessFunction {\n    id\n    name\n    description\n    runtime\n    timeoutSeconds\n    syncStatus\n    latestVersion\n    latestVersionInputSchema\n    publishedVersions\n    createdAt\n    updatedAt\n  }\n": types.ServerlessFunctionFieldsFragmentDoc,
     "\n  \n  mutation CreateOneServerlessFunctionItem(\n    $input: CreateServerlessFunctionInput!\n  ) {\n    createOneServerlessFunction(input: $input) {\n      ...ServerlessFunctionFields\n    }\n  }\n": types.CreateOneServerlessFunctionItemDocument,
     "\n  \n  mutation DeleteOneServerlessFunction($input: ServerlessFunctionIdInput!) {\n    deleteOneServerlessFunction(input: $input) {\n      ...ServerlessFunctionFields\n    }\n  }\n": types.DeleteOneServerlessFunctionDocument,
     "\n  mutation ExecuteOneServerlessFunction(\n    $input: ExecuteServerlessFunctionInput!\n  ) {\n    executeOneServerlessFunction(input: $input) {\n      data\n      duration\n      status\n      error\n    }\n  }\n": types.ExecuteOneServerlessFunctionDocument,
@@ -142,7 +142,7 @@ export function graphql(source: "\n  query ObjectMetadataItems(\n    $objectFilt
 /**
  * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
  */
-export function graphql(source: "\n  fragment ServerlessFunctionFields on ServerlessFunction {\n    id\n    name\n    description\n    runtime\n    syncStatus\n    latestVersion\n    latestVersionInputSchema\n    publishedVersions\n    createdAt\n    updatedAt\n  }\n"): (typeof documents)["\n  fragment ServerlessFunctionFields on ServerlessFunction {\n    id\n    name\n    description\n    runtime\n    syncStatus\n    latestVersion\n    latestVersionInputSchema\n    publishedVersions\n    createdAt\n    updatedAt\n  }\n"];
+export function graphql(source: "\n  fragment ServerlessFunctionFields on ServerlessFunction {\n    id\n    name\n    description\n    runtime\n    timeoutSeconds\n    syncStatus\n    latestVersion\n    latestVersionInputSchema\n    publishedVersions\n    createdAt\n    updatedAt\n  }\n"): (typeof documents)["\n  fragment ServerlessFunctionFields on ServerlessFunction {\n    id\n    name\n    description\n    runtime\n    timeoutSeconds\n    syncStatus\n    latestVersion\n    latestVersionInputSchema\n    publishedVersions\n    createdAt\n    updatedAt\n  }\n"];
 /**
  * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
  */

packages/twenty-front/src/generated-metadata/graphql.ts

@@ -2314,4 +2314,4 @@ export const UpdateOneServerlessFunctionDocument = {"kind":"Document","definitio
 export const FindManyAvailablePackagesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindManyAvailablePackages"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunctionIdInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getAvailablePackages"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}]}]}}]} as unknown as DocumentNode<FindManyAvailablePackagesQuery, FindManyAvailablePackagesQueryVariables>;
 export const GetManyServerlessFunctionsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetManyServerlessFunctions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"findManyServerlessFunctions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersionInputSchema"}},{"kind":"Field","name":{"kind":"Name","value":"publishedVersions"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<GetManyServerlessFunctionsQuery, GetManyServerlessFunctionsQueryVariables>;
 export const GetOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunctionIdInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"findOneServerlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersionInputSchema"}},{"kind":"Field","name":{"kind":"Name","value":"publishedVersions"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<GetOneServerlessFunctionQuery, GetOneServerlessFunctionQueryVariables>;
-export const FindOneServerlessFunctionSourceCodeDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindOneServerlessFunctionSourceCode"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"GetServerlessFunctionSourceCodeInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getServerlessFunctionSourceCode"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}]}]}}]} as unknown as DocumentNode<FindOneServerlessFunctionSourceCodeQuery, FindOneServerlessFunctionSourceCodeQueryVariables>;
+export const FindOneServerlessFunctionSourceCodeDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindOneServerlessFunctionSourceCode"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"GetServerlessFunctionSourceCodeInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getServerlessFunctionSourceCode"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}]}]}}]} as unknown as DocumentNode<FindOneServerlessFunctionSourceCodeQuery, FindOneServerlessFunctionSourceCodeQueryVariables>;

packages/twenty-front/src/generated/graphql.tsx

@@ -223,6 +223,7 @@ export type CreateOneFieldMetadataInput = {
 export type CreateServerlessFunctionInput = {
   description?: InputMaybe<Scalars['String']>;
   name: Scalars['String'];
+  timeoutSeconds?: InputMaybe<Scalars['Float']>;
 };
 
 export type CreateWorkflowVersionStepInput = {
@@ -326,12 +327,12 @@ export enum FeatureFlagKey {
   IsFunctionSettingsEnabled = 'IsFunctionSettingsEnabled',
   IsGmailSendEmailScopeEnabled = 'IsGmailSendEmailScopeEnabled',
   IsJsonFilterEnabled = 'IsJsonFilterEnabled',
+  IsLocalizationEnabled = 'IsLocalizationEnabled',
   IsMicrosoftSyncEnabled = 'IsMicrosoftSyncEnabled',
   IsPostgreSqlIntegrationEnabled = 'IsPostgreSQLIntegrationEnabled',
   IsStripeIntegrationEnabled = 'IsStripeIntegrationEnabled',
   IsUniqueIndexesEnabled = 'IsUniqueIndexesEnabled',
-  IsWorkflowEnabled = 'IsWorkflowEnabled',
+  IsWorkflowEnabled = 'IsWorkflowEnabled'
-  IsLocalizationEnabled = 'IsLocalizationEnabled'
 }
 
 export type FieldConnection = {
@@ -1139,6 +1140,7 @@ export type ServerlessFunction = {
   publishedVersions: Array<Scalars['String']>;
   runtime: Scalars['String'];
   syncStatus: ServerlessFunctionSyncStatus;
+  timeoutSeconds: Scalars['Float'];
   updatedAt: Scalars['DateTime'];
 };
 
@@ -1390,6 +1392,7 @@ export type UpdateServerlessFunctionInput = {
   /** Id of the serverless function to execute */
   id: Scalars['UUID'];
   name: Scalars['String'];
+  timeoutSeconds?: InputMaybe<Scalars['Float']>;
 };
 
 export type UpdateWorkflowVersionStepInput = {
@@ -4705,4 +4708,4 @@ export function useGetWorkspaceFromInviteHashLazyQuery(baseOptions?: Apollo.Lazy
         }
 export type GetWorkspaceFromInviteHashQueryHookResult = ReturnType<typeof useGetWorkspaceFromInviteHashQuery>;
 export type GetWorkspaceFromInviteHashLazyQueryHookResult = ReturnType<typeof useGetWorkspaceFromInviteHashLazyQuery>;
-export type GetWorkspaceFromInviteHashQueryResult = Apollo.QueryResult<GetWorkspaceFromInviteHashQuery, GetWorkspaceFromInviteHashQueryVariables>;
+export type GetWorkspaceFromInviteHashQueryResult = Apollo.QueryResult<GetWorkspaceFromInviteHashQuery, GetWorkspaceFromInviteHashQueryVariables>;

packages/twenty-front/src/modules/settings/serverless-functions/graphql/fragments/serverlessFunctionFragment.ts

@@ -6,6 +6,7 @@ export const SERVERLESS_FUNCTION_FRAGMENT = gql`
     name
     description
     runtime
+    timeoutSeconds
     syncStatus
     latestVersion
     latestVersionInputSchema

packages/twenty-server/src/database/typeorm/metadata/migrations/1737047131108-addTimeoutSecondsColumnToServerless.ts

@@ -0,0 +1,25 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddTimeoutSecondsColumnToServerless1737047131108
+  implements MigrationInterface
+{
+  name = 'AddTimeoutSecondsColumnToServerless1737047131108';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "metadata"."serverlessFunction" ADD "timeoutSeconds" integer NOT NULL DEFAULT '300'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "metadata"."serverlessFunction" ADD CONSTRAINT "CHK_4a5179975ee017934a91703247" CHECK ("timeoutSeconds" >= 1 AND "timeoutSeconds" <= 900)`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "metadata"."serverlessFunction" DROP CONSTRAINT "CHK_4a5179975ee017934a91703247"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "metadata"."serverlessFunction" DROP COLUMN "timeoutSeconds"`,
+    );
+  }
+}

packages/twenty-server/src/engine/core-modules/serverless/drivers/lambda.driver.ts

@@ -237,7 +237,7 @@ export class LambdaDriver implements ServerlessDriver {
         Role: this.lambdaRole,
         Runtime: serverlessFunction.runtime,
         Description: 'Lambda function to run user script',
-        Timeout: 900,
+        Timeout: serverlessFunction.timeoutSeconds,
       };
 
       const command = new CreateFunctionCommand(params);
@@ -259,6 +259,7 @@ export class LambdaDriver implements ServerlessDriver {
             Variables: envVariables,
           },
           FunctionName: serverlessFunction.id,
+          Timeout: serverlessFunction.timeoutSeconds,
         };
 
       const updateConfigurationCommand = new UpdateFunctionConfigurationCommand(

packages/twenty-server/src/engine/core-modules/serverless/drivers/local.driver.ts

@@ -183,7 +183,17 @@ export class LocalDriver implements ServerlessDriver {
       return await new Promise((resolve, reject) => {
         const child = fork(listenerFile, { silent: true });
 
+        const timeoutMs = serverlessFunction.timeoutSeconds * 1_000;
+
+        const timeoutHandler = setTimeout(() => {
+          child.kill();
+          const duration = Date.now() - startTime;
+
+          reject(new Error(`Task timed out after ${duration / 1_000} seconds`));
+        }, timeoutMs);
+
         child.on('message', (message: object | ServerlessExecuteError) => {
+          clearTimeout(timeoutHandler);
           const duration = Date.now() - startTime;
 
           if ('errorType' in message) {
@@ -204,6 +214,7 @@ export class LocalDriver implements ServerlessDriver {
         });
 
         child.stderr?.on('data', (data) => {
+          clearTimeout(timeoutHandler);
           const stackTrace = data
             .toString()
             .split('\n')
@@ -235,11 +246,13 @@ export class LocalDriver implements ServerlessDriver {
         });
 
         child.on('error', (error) => {
+          clearTimeout(timeoutHandler);
           reject(error);
           child.kill();
         });
 
         child.on('exit', (code) => {
+          clearTimeout(timeoutHandler);
           if (code && code !== 0) {
             reject(new Error(`Child process exited with code ${code}`));
           }

packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input.ts

@@ -1,6 +1,13 @@
 import { Field, InputType } from '@nestjs/graphql';
 
-import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
+import {
+  IsNotEmpty,
+  IsNumber,
+  IsOptional,
+  IsString,
+  Max,
+  Min,
+} from 'class-validator';
 
 @InputType()
 export class CreateServerlessFunctionInput {
@@ -13,4 +20,11 @@ export class CreateServerlessFunctionInput {
   @IsOptional()
   @Field({ nullable: true })
   description?: string;
+
+  @IsNumber()
+  @Field({ nullable: true })
+  @Min(1)
+  @Max(900)
+  @IsOptional()
+  timeoutSeconds?: number;
 }

packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts

@@ -15,6 +15,7 @@ import {
   IsDateString,
   IsEnum,
   IsNotEmpty,
+  IsNumber,
   IsString,
   IsUUID,
 } from 'class-validator';
@@ -58,6 +59,10 @@ export class ServerlessFunctionDTO {
   @Field()
   runtime: string;
 
+  @IsNumber()
+  @Field()
+  timeoutSeconds: number;
+
   @IsString()
   @Field({ nullable: true })
   latestVersion: string;

packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input.ts

@@ -1,6 +1,15 @@
 import { Field, InputType } from '@nestjs/graphql';
 
-import { IsNotEmpty, IsObject, IsString, IsUUID } from 'class-validator';
+import {
+  IsNotEmpty,
+  IsNumber,
+  IsObject,
+  IsOptional,
+  IsString,
+  IsUUID,
+  Max,
+  Min,
+} from 'class-validator';
 import graphqlTypeJson from 'graphql-type-json';
 
 import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@@ -20,8 +29,16 @@ export class UpdateServerlessFunctionInput {
 
   @IsString()
   @Field({ nullable: true })
+  @IsOptional()
   description?: string;
 
+  @IsNumber()
+  @Field({ nullable: true })
+  @Min(1)
+  @Max(900)
+  @IsOptional()
+  timeoutSeconds?: number;
+
   @Field(() => graphqlTypeJson)
   @IsObject()
   code: JSON;

packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts

@@ -1,4 +1,5 @@
 import {
+  Check,
   Column,
   CreateDateColumn,
   Entity,
@@ -8,6 +9,8 @@ import {
 
 import { InputSchema } from 'src/modules/workflow/workflow-builder/types/input-schema.type';
 
+const DEFAULT_SERVERLESS_TIMEOUT_SECONDS = 300; // 5 minutes
+
 export enum ServerlessFunctionSyncStatus {
   NOT_READY = 'NOT_READY',
   READY = 'READY',
@@ -40,6 +43,10 @@ export class ServerlessFunctionEntity {
   @Column({ nullable: false, default: ServerlessFunctionRuntime.NODE18 })
   runtime: ServerlessFunctionRuntime;
 
+  @Column({ nullable: false, default: DEFAULT_SERVERLESS_TIMEOUT_SECONDS })
+  @Check(`"timeoutSeconds" >= 1 AND "timeoutSeconds" <= 900`)
+  timeoutSeconds: number;
+
   @Column({ nullable: true })
   layerVersion: number;
 

packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.resolver.ts

@@ -159,10 +159,7 @@ export class ServerlessFunctionResolver {
       await this.checkFeatureFlag(workspaceId);
 
       return await this.serverlessFunctionService.createOneServerlessFunction(
-        {
+        input,
-          name: input.name,
-          description: input.description,
-        },
         workspaceId,
       );
     } catch (error) {

packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts

@@ -273,6 +273,7 @@ export class ServerlessFunctionService {
         name: serverlessFunctionInput.name,
         description: serverlessFunctionInput.description,
         syncStatus: ServerlessFunctionSyncStatus.NOT_READY,
+        timeoutSeconds: serverlessFunctionInput.timeoutSeconds,
       },
     );
 
@@ -393,6 +394,7 @@ export class ServerlessFunctionService {
       {
         name: serverlessFunctionToCopy?.name,
         description: serverlessFunctionToCopy?.description,
+        timeoutSeconds: serverlessFunctionToCopy?.timeoutSeconds,
         workspaceId,
         layerVersion: LAST_LAYER_VERSION,
       },