admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add workflow runner (#6458)

Browse Source 
- add workflow runner module
- add an endpoint to trigger a workflow via api
- improve error handling for serverless functions

## Testing
- create 2 serverless functions
- create a workflow
- create this workflow Version
```
{
  "type": "MANUAL",
  "input": {"b": "bb"},
  "nextAction": {
    "name": "step_1",
    "displayName": "Code",
    "type": "CODE",
    "valid": true,
    "settings": {
      "serverlessFunctionId": "Serverless function 1 Id",
      "errorHandlingOptions": {
        "retryOnFailure": {
          "value": false
        },
        "continueOnFailure": {
          "value": false
        }
      }
    },
    "nextAction": {
      "name": "step_1",
      "displayName": "Code",
      "type": "CODE",
      "valid": true,
      "settings": {
        "serverlessFunctionId": "Serverless function 1 Id",
        "errorHandlingOptions": {
          "retryOnFailure": {
            "value": false
          },
          "continueOnFailure": {
            "value": false
          }
        }
      },
      "nextAction": {
        "name": "step_1",
        "displayName": "Code",
        "type": "CODE",
        "valid": true,
        "settings": {
          "serverlessFunctionId": "Serverless function 2 Id",
          "errorHandlingOptions": {
            "retryOnFailure": {
              "value": false
            },
            "continueOnFailure": {
              "value": false
            }
          }
        }
      }
    }
  }
}

`
``
- call 
```
mutation Trigger {
  triggerWorkflow(workflowVersionId: "WORKFLOW_VERSION_ID") {
    result
  }
}
```
- try when errors are injected in serverless function
This commit is contained in:
martmull
2024-07-31 12:48:33 +02:00
committed by GitHub
parent b8496d22b6
commit 6b4c79ff0d
42 changed files with 639 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
packages/twenty-server/src/engine/core-modules/workflow/dtos/workflow-trigger-result.dto.ts Normal file
View File

@ -0,0 +1,14 @@
import { Field, ObjectType } from '@nestjs/graphql';
import { IsObject } from 'class-validator';
import graphqlTypeJson from 'graphql-type-json';
@ObjectType('WorkflowTriggerResult')
export class WorkflowTriggerResultDTO {
@IsObject()
@Field(() => graphqlTypeJson, {
description: 'Execution result in JSON format',
nullable: true,
})
result?: JSON;
}

1
packages/twenty-server/src/engine/core-modules/workflow/utils/workflow-trigger-graphql-api-exception-handler.util.ts
View File

@ -14,6 +14,7 @@ export const workflowTriggerGraphqlApiExceptionHandler = (error: Error) => {
throw new UserInputError(error.message);
case WorkflowTriggerExceptionCode.INVALID_WORKFLOW_TRIGGER:
case WorkflowTriggerExceptionCode.INVALID_WORKFLOW_VERSION:
case WorkflowTriggerExceptionCode.INVALID_ACTION_TYPE:
default:
throw new InternalServerError(error.message);
}

3
packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger.module.ts
View File

@ -1,9 +1,12 @@
import { Module } from '@nestjs/common';
import { WorkflowTriggerResolver } from 'src/engine/core-modules/workflow/workflow-trigger.resolver';
import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-common.module';
import { WorkflowTriggerService } from 'src/modules/workflow/workflow-trigger/workflow-trigger.service';
import { WorkflowRunnerModule } from 'src/modules/workflow/workflow-runner/workflow-runner.module';
@Module({
imports: [WorkflowCommonModule, WorkflowRunnerModule],
providers: [WorkflowTriggerService, WorkflowTriggerResolver],
})
export class WorkflowTriggerModule {}

18
packages/twenty-server/src/engine/core-modules/workflow/workflow-trigger.resolver.ts
View File

@ -6,6 +6,7 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { WorkflowTriggerService } from 'src/modules/workflow/workflow-trigger/workflow-trigger.service';
import { WorkflowTriggerResultDTO } from 'src/engine/core-modules/workflow/dtos/workflow-trigger-result.dto';
@UseGuards(JwtAuthGuard)
@Resolver()
@ -28,4 +29,21 @@ export class WorkflowTriggerResolver {
workflowTriggerGraphqlApiExceptionHandler(error);
}
}
@Mutation(() => WorkflowTriggerResultDTO)
async triggerWorkflow(
@AuthWorkspace() { id: workspaceId }: Workspace,
@Args('workflowVersionId') workflowVersionId: string,
) {
try {
return {
result: await this.workflowTriggerService.runWorkflow(
workspaceId,
workflowVersionId,
),
};
} catch (error) {
workflowTriggerGraphqlApiExceptionHandler(error);
}
}
}

1
packages/twenty-server/src/engine/integrations/message-queue/message-queue.constants.ts
View File

@ -16,4 +16,5 @@ export enum MessageQueue {
recordPositionBackfillQueue = 'record-position-backfill-queue',
entityEventsToDbQueue = 'entity-events-to-db-queue',
testQueue = 'test-queue',
workflowQueue = 'workflow-queue',
}

16
packages/twenty-server/src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface.ts
View File

@ -1,4 +1,18 @@
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
export type ServerlessExecuteError = {
errorType: string;
errorMessage: string;
stackTrace: string;
};
export type ServerlessExecuteResult = {
data: object | null;
duration: number;
status: ServerlessFunctionExecutionStatus;
error?: ServerlessExecuteError;
};
export interface ServerlessDriver {
delete(serverlessFunction: ServerlessFunctionEntity): Promise<void>;
@ -6,5 +20,5 @@ export interface ServerlessDriver {
execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined,
): Promise<object>;
): Promise<ServerlessExecuteResult>;
}

30
packages/twenty-server/src/engine/integrations/serverless/drivers/lambda.driver.ts
View File

@ -13,13 +13,17 @@ import {
import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
import { UpdateFunctionCodeCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/UpdateFunctionCodeCommand';
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import {
ServerlessDriver,
ServerlessExecuteResult,
} from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import { createZipFile } from 'src/engine/integrations/serverless/drivers/utils/create-zip-file';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
export interface LambdaDriverOptions extends LambdaClientConfig {
fileStorageService: FileStorageService;
@ -134,7 +138,8 @@ export class LambdaDriver
async execute(
functionToExecute: ServerlessFunctionEntity,
payload: object | undefined = undefined,
): Promise<object> {
): Promise<ServerlessExecuteResult> {
const startTime = Date.now();
const params = {
FunctionName: functionToExecute.id,
Payload: JSON.stringify(payload),
@ -144,10 +149,25 @@ export class LambdaDriver
const result = await this.lambdaClient.send(command);
if (!result.Payload) {
return {};
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
const duration = Date.now() - startTime;
if (result.FunctionError) {
return {
data: null,
duration,
status: ServerlessFunctionExecutionStatus.ERROR,
error: parsedResult,
};
}
return JSON.parse(result.Payload.transformToString());
return {
data: parsedResult,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
};
}
}

45
packages/twenty-server/src/engine/integrations/serverless/drivers/local.driver.ts
View File

@ -5,13 +5,18 @@ import { fork } from 'child_process';
import { v4 } from 'uuid';
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import {
ServerlessDriver,
ServerlessExecuteError,
ServerlessExecuteResult,
} from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/integrations/file-storage/utils/read-file-content';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { BUILD_FILE_NAME } from 'src/engine/integrations/serverless/drivers/constants/build-file-name';
import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
@ -51,7 +56,8 @@ export class LocalDriver
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined = undefined,
): Promise<object> {
): Promise<ServerlessExecuteResult> {
const startTime = Date.now();
const fileStream = await this.fileStorageService.read({
folderPath: this.getFolderPath(serverlessFunction),
filename: BUILD_FILE_NAME,
@ -83,8 +89,23 @@ export class LocalDriver
return await new Promise((resolve, reject) => {
const child = fork(tmpFilePath, { silent: true });
child.on('message', (message: object) => {
resolve(message);
child.on('message', (message: object | ServerlessExecuteError) => {
const duration = Date.now() - startTime;
if ('errorType' in message) {
resolve({
data: null,
duration,
error: message,
status: ServerlessFunctionExecutionStatus.ERROR,
});
} else {
resolve({
data: message,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
});
}
child.kill();
fs.unlink(tmpFilePath);
});
@ -93,8 +114,8 @@ export class LocalDriver
const stackTrace = data
.toString()
.split('\n')
.filter((line) => line.trim() !== '');
const errorTrace = stackTrace.filter((line) =>
.filter((line: string) => line.trim() !== '');
const errorTrace = stackTrace.filter((line: string) =>
line.includes('Error: '),
)?.[0];
@ -105,11 +126,17 @@ export class LocalDriver
errorType = errorTrace.split(':')[0];
errorMessage = errorTrace.split(': ')[1];
}
const duration = Date.now() - startTime;
resolve({
errorType,
errorMessage,
stackTrace: stackTrace,
data: null,
duration,
status: ServerlessFunctionExecutionStatus.ERROR,
error: {
errorType,
errorMessage,
stackTrace: stackTrace,
},
});
child.kill();
fs.unlink(tmpFilePath);

7
packages/twenty-server/src/engine/integrations/serverless/serverless.service.ts
View File

@ -1,6 +1,9 @@
import { Inject, Injectable } from '@nestjs/common';
import { ServerlessDriver } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import {
ServerlessDriver,
ServerlessExecuteResult,
} from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import { SERVERLESS_DRIVER } from 'src/engine/integrations/serverless/serverless.constants';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
@ -20,7 +23,7 @@ export class ServerlessService implements ServerlessDriver {
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined = undefined,
) {
): Promise<ServerlessExecuteResult> {
return this.driver.execute(serverlessFunction, payload);
}
}

39
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto.ts
View File

@ -1,13 +1,44 @@
import { Field, ObjectType } from '@nestjs/graphql';
import { Field, ObjectType, registerEnumType } from '@nestjs/graphql';
import { IsObject } from 'class-validator';
import { IsObject, IsOptional } from 'class-validator';
import graphqlTypeJson from 'graphql-type-json';
export enum ServerlessFunctionExecutionStatus {
SUCCESS = 'SUCCESS',
ERROR = 'ERROR',
}
registerEnumType(ServerlessFunctionExecutionStatus, {
name: 'ServerlessFunctionExecutionStatus',
description: 'Status of the serverless function execution',
});
@ObjectType('ServerlessFunctionExecutionResult')
export class ServerlessFunctionExecutionResultDto {
export class ServerlessFunctionExecutionResultDTO {
@IsObject()
@Field(() => graphqlTypeJson, {
description: 'Execution result in JSON format',
nullable: true,
})
result: JSON;
data?: JSON;
@Field({ description: 'Execution duration in milliseconds' })
duration: number;
@Field(() => ServerlessFunctionExecutionStatus, {
description: 'Execution status',
})
status: ServerlessFunctionExecutionStatus;
@IsObject()
@IsOptional()
@Field(() => graphqlTypeJson, {
description: 'Execution error in JSON format',
nullable: true,
})
error?: {
errorType: string;
errorMessage: string;
stackTrace: string;
};
}

2
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts
View File

@ -36,7 +36,7 @@ registerEnumType(ServerlessFunctionSyncStatus, {
defaultResultSize: 10,
maxResultsSize: 1000,
})
export class ServerlessFunctionDto {
export class ServerlessFunctionDTO {
@IsUUID()
@IsNotEmpty()
@IDField(() => UUIDScalarType)

4
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.module.ts
View File

@ -13,7 +13,7 @@ import { ServerlessModule } from 'src/engine/integrations/serverless/serverless.
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { ServerlessFunctionResolver } from 'src/engine/metadata-modules/serverless-function/serverless-function.resolver';
import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard';
import { ServerlessFunctionDto } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
import { ServerlessFunctionDTO } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
import { FileUploadModule } from 'src/engine/core-modules/file/file-upload/file-upload.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
@ -32,7 +32,7 @@ import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-
resolvers: [
{
EntityClass: ServerlessFunctionEntity,
DTOClass: ServerlessFunctionDto,
DTOClass: ServerlessFunctionDTO,
ServiceClass: ServerlessFunctionService,
pagingStrategy: PagingStrategies.CURSOR,
read: {

26
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.resolver.ts
View File

@ -14,8 +14,8 @@ import { CreateServerlessFunctionFromFileInput } from 'src/engine/metadata-modul
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { DeleteServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/delete-serverless-function.input';
import { ExecuteServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input';
import { ServerlessFunctionExecutionResultDto } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import { ServerlessFunctionDto } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
import { ServerlessFunctionExecutionResultDTO } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import { ServerlessFunctionDTO } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto';
import { UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input';
import {
ServerlessFunctionException,
@ -49,7 +49,7 @@ export class ServerlessFunctionResolver {
}
}
@Mutation(() => ServerlessFunctionDto)
@Mutation(() => ServerlessFunctionDTO)
async deleteOneServerlessFunction(
@Args('input') input: DeleteServerlessFunctionInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
@ -66,7 +66,7 @@ export class ServerlessFunctionResolver {
}
}
@Mutation(() => ServerlessFunctionDto)
@Mutation(() => ServerlessFunctionDTO)
async updateOneServerlessFunction(
@Args('input')
input: UpdateServerlessFunctionInput,
@ -84,7 +84,7 @@ export class ServerlessFunctionResolver {
}
}
@Mutation(() => ServerlessFunctionDto)
@Mutation(() => ServerlessFunctionDTO)
async createOneServerlessFunction(
@Args('input')
input: CreateServerlessFunctionInput,
@ -106,7 +106,7 @@ export class ServerlessFunctionResolver {
}
}
@Mutation(() => ServerlessFunctionDto)
@Mutation(() => ServerlessFunctionDTO)
async createOneServerlessFunctionFromFile(
@Args({ name: 'file', type: () => GraphQLUpload })
file: FileUpload,
@ -127,7 +127,7 @@ export class ServerlessFunctionResolver {
}
}
@Mutation(() => ServerlessFunctionExecutionResultDto)
@Mutation(() => ServerlessFunctionExecutionResultDTO)
async executeOneServerlessFunction(
@Args() executeServerlessFunctionInput: ExecuteServerlessFunctionInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
@ -136,13 +136,11 @@ export class ServerlessFunctionResolver {
await this.checkFeatureFlag(workspaceId);
const { id, payload } = executeServerlessFunctionInput;
return {
result: await this.serverlessFunctionService.executeOne(
id,
workspaceId,
payload,
),
};
return await this.serverlessFunctionService.executeOne(
id,
workspaceId,
payload,
);
} catch (error) {
serverlessFunctionGraphQLApiExceptionHandler(error);
}

3
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
View File

@ -9,6 +9,7 @@ import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
import { v4 } from 'uuid';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { ServerlessExecuteResult } from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import { ServerlessService } from 'src/engine/integrations/serverless/serverless.service';
import {
@ -41,7 +42,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
id: string,
workspaceId: string,
payload: object | undefined = undefined,
) {
): Promise<ServerlessExecuteResult> {
const functionToExecute = await this.serverlessFunctionRepository.findOne({
where: {
id,

6
packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/standard-objects/index.ts
View File

@ -26,9 +26,9 @@ import { ViewFilterWorkspaceEntity } from 'src/modules/view/standard-objects/vie
import { ViewSortWorkspaceEntity } from 'src/modules/view/standard-objects/view-sort.workspace-entity';
import { ViewWorkspaceEntity } from 'src/modules/view/standard-objects/view.workspace-entity';
import { WebhookWorkspaceEntity } from 'src/modules/webhook/standard-objects/webhook.workspace-entity';
import { WorkflowEventListenerWorkspaceEntity } from 'src/modules/workflow/standard-objects/workflow-event-listener.workspace-entity';
import { WorkflowVersionWorkspaceEntity } from 'src/modules/workflow/standard-objects/workflow-version.workspace-entity';
import { WorkflowWorkspaceEntity } from 'src/modules/workflow/standard-objects/workflow.workspace-entity';
import { WorkflowEventListenerWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-event-listener.workspace-entity';
import { WorkflowVersionWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow-version.workspace-entity';
import { WorkflowWorkspaceEntity } from 'src/modules/workflow/common/standard-objects/workflow.workspace-entity';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
// TODO: Maybe we should automate this with the DiscoverService of Nest.JS