admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Infer function input in workflow step (#8308)

Browse Source 
- add `inputSchema` column in serverless function. This is an array of
parameters, with their name and type
- on serverless function id update, get the `inputSchema` + store empty
settings in step
- from step settings, build the form 

TODO in next PR:
- use field type to decide what kind of form should be printed
- have a strategy to handle object as input



https://github.com/user-attachments/assets/ed96f919-24b5-4baf-a051-31f76f45e575
This commit is contained in:
Thomas Trompette
2024-11-05 14:57:06 +01:00
committed by GitHub
parent d1531aa1b6
commit be8141ce5e
29 changed files with 334 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
packages/twenty-server/src/database/typeorm/metadata/migrations/1730803174864-addInputSchemaToFunction.ts Normal file
View File

@ -0,0 +1,19 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddInputSchemaToFunction1730803174864
implements MigrationInterface
{
name = 'AddInputSchemaToFunction1730803174864';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."serverlessFunction" ADD "latestVersionInputSchema" jsonb`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."serverlessFunction" DROP COLUMN "latestVersionInputSchema"`,
);
}
}

6
packages/twenty-server/src/engine/core-modules/workflow/resolvers/workflow-builder.resolver.ts
View File

@ -9,7 +9,7 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { WorkflowBuilderService } from 'src/modules/workflow/workflow-builder/workflow-builder.service';
import { WorkflowBuilderWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-builder.workspace-service';
import { OutputSchema } from 'src/modules/workflow/workflow-executor/types/workflow-step-settings.type';
@Resolver()
@ -17,7 +17,7 @@ import { OutputSchema } from 'src/modules/workflow/workflow-executor/types/workf
@UseFilters(WorkflowTriggerGraphqlApiExceptionFilter)
export class WorkflowBuilderResolver {
constructor(
private readonly workflowBuilderService: WorkflowBuilderService,
private readonly workflowBuilderWorkspaceService: WorkflowBuilderWorkspaceService,
) {}
@Mutation(() => graphqlTypeJson)
@ -25,7 +25,7 @@ export class WorkflowBuilderResolver {
@AuthWorkspace() { id: workspaceId }: Workspace,
@Args('input') { step }: ComputeStepOutputSchemaInput,
): Promise<OutputSchema> {
return this.workflowBuilderService.computeStepOutputSchema({
return this.workflowBuilderWorkspaceService.computeStepOutputSchema({
step,
workspaceId,
});

1
packages/twenty-server/src/engine/metadata-modules/serverless-function/constants/serverless-function-published.ts Normal file
View File

@ -0,0 +1 @@
export const SERVERLESS_FUNCTION_PUBLISHED = 'serverlessFunction.published';

14
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/function-parameter.dto.ts Normal file
View File

@ -0,0 +1,14 @@
import { Field, ObjectType } from '@nestjs/graphql';
import { IsString } from 'class-validator';
@ObjectType()
export class FunctionParameter {
@IsString()
@Field(() => String)
name: string;
@IsString()
@Field(() => String)
type: string;
}

5
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts
View File

@ -20,6 +20,7 @@ import {
} from 'class-validator';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
import { FunctionParameter } from 'src/engine/metadata-modules/serverless-function/dtos/function-parameter.dto';
import { ServerlessFunctionSyncStatus } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
registerEnumType(ServerlessFunctionSyncStatus, {
@ -64,6 +65,10 @@ export class ServerlessFunctionDTO {
@Field(() => [String], { nullable: false })
publishedVersions: string[];
@IsArray()
@Field(() => [FunctionParameter], { nullable: true })
latestVersionInputSchema: FunctionParameter[] | null;
@IsEnum(ServerlessFunctionSyncStatus)
@IsNotEmpty()
@Field(() => ServerlessFunctionSyncStatus)

59
packages/twenty-server/src/engine/metadata-modules/serverless-function/listeners/serverless-function-publication.listener.ts Normal file
View File

@ -0,0 +1,59 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { InjectRepository } from '@nestjs/typeorm';
import { join } from 'path';
import { Repository } from 'typeorm';
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
import { SERVERLESS_FUNCTION_PUBLISHED } from 'src/engine/metadata-modules/serverless-function/constants/serverless-function-published';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/workspace-event.type';
import { CodeIntrospectionService } from 'src/modules/code-introspection/code-introspection.service';
@Injectable()
export class ServerlessFunctionPublicationListener {
constructor(
private readonly serverlessFunctionService: ServerlessFunctionService,
private readonly codeIntrospectionService: CodeIntrospectionService,
@InjectRepository(ServerlessFunctionEntity, 'metadata')
private readonly serverlessFunctionRepository: Repository<ServerlessFunctionEntity>,
) {}
@OnEvent(SERVERLESS_FUNCTION_PUBLISHED)
async handle(
payload: WorkspaceEventBatch<{
serverlessFunctionId: string;
serverlessFunctionVersion: string;
}>,
): Promise<void> {
payload.events.forEach(async (event) => {
const sourceCode =
await this.serverlessFunctionService.getServerlessFunctionSourceCode(
payload.workspaceId,
event.serverlessFunctionId,
event.serverlessFunctionVersion,
);
if (!sourceCode) {
return;
}
const indexCode = sourceCode[join('src', INDEX_FILE_NAME)];
if (!indexCode) {
return;
}
const latestVersionInputSchema =
await this.codeIntrospectionService.getFunctionInputSchema(indexCode);
await this.serverlessFunctionRepository.update(
{ id: event.serverlessFunctionId },
{ latestVersionInputSchema },
);
});
}
}

5
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts
View File

@ -6,6 +6,8 @@ import {
UpdateDateColumn,
} from 'typeorm';
import { FunctionParameter } from 'src/engine/metadata-modules/serverless-function/dtos/function-parameter.dto';
export enum ServerlessFunctionSyncStatus {
NOT_READY = 'NOT_READY',
READY = 'READY',
@ -32,6 +34,9 @@ export class ServerlessFunctionEntity {
@Column({ nullable: false, type: 'jsonb', default: [] })
publishedVersions: string[];
@Column({ nullable: true, type: 'jsonb' })
latestVersionInputSchema: FunctionParameter[];
@Column({ nullable: false, default: ServerlessFunctionRuntime.NODE18 })
runtime: ServerlessFunctionRuntime;

9
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.module.ts
View File

@ -8,9 +8,11 @@ import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-
import { FileUploadModule } from 'src/engine/core-modules/file/file-upload/file-upload.module';
import { FileModule } from 'src/engine/core-modules/file/file.module';
import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
import { ServerlessFunctionPublicationListener } from 'src/engine/metadata-modules/serverless-function/listeners/serverless-function-publication.listener';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { ServerlessFunctionResolver } from 'src/engine/metadata-modules/serverless-function/serverless-function.resolver';
import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service';
import { CodeIntrospectionModule } from 'src/modules/code-introspection/code-introspection.module';
@Module({
imports: [
@ -20,8 +22,13 @@ import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverles
FileModule,
ThrottlerModule,
AnalyticsModule,
CodeIntrospectionModule,
],
providers: [
ServerlessFunctionService,
ServerlessFunctionResolver,
ServerlessFunctionPublicationListener,
],
providers: [ServerlessFunctionService, ServerlessFunctionResolver],
exports: [ServerlessFunctionService],
})
export class ServerlessFunctionModule {}

14
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
View File

@ -21,6 +21,7 @@ import { getLastLayerDependencies } from 'src/engine/core-modules/serverless/dri
import { ServerlessService } from 'src/engine/core-modules/serverless/serverless.service';
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
import { SERVERLESS_FUNCTION_PUBLISHED } from 'src/engine/metadata-modules/serverless-function/constants/serverless-function-published';
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input';
import {
@ -31,6 +32,7 @@ import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
import { isDefined } from 'src/utils/is-defined';
@Injectable()
@ -43,6 +45,7 @@ export class ServerlessFunctionService {
private readonly throttlerService: ThrottlerService,
private readonly environmentService: EnvironmentService,
private readonly analyticsService: AnalyticsService,
private readonly workspaceEventEmitter: WorkspaceEventEmitter,
) {}
async findManyServerlessFunctions(where) {
@ -191,6 +194,17 @@ export class ServerlessFunctionService {
},
);
this.workspaceEventEmitter.emit(
SERVERLESS_FUNCTION_PUBLISHED,
[
{
serverlessFunctionId: existingServerlessFunction.id,
serverlessFunctionVersion: newVersion,
},
],
workspaceId,
);
return this.serverlessFunctionRepository.findOneBy({
id: existingServerlessFunction.id,
});

18
packages/twenty-server/src/modules/code-introspection/__tests__/code-introspection.service.spec.ts
View File

@ -18,7 +18,7 @@ describe('CodeIntrospectionService', () => {
expect(service).toBeDefined();
});
describe('analyze', () => {
describe('getFunctionInputSchema', () => {
it('should analyze a function declaration correctly', () => {
const fileContent = `
function testFunction(param1: string, param2: number): void {
@ -26,7 +26,7 @@ describe('CodeIntrospectionService', () => {
}
`;
const result = service.analyze(fileContent);
const result = service.getFunctionInputSchema(fileContent);
expect(result).toEqual([
{ name: 'param1', type: 'string' },
@ -41,7 +41,7 @@ describe('CodeIntrospectionService', () => {
};
`;
const result = service.analyze(fileContent);
const result = service.getFunctionInputSchema(fileContent);
expect(result).toEqual([
{ name: 'param1', type: 'string' },
@ -55,7 +55,7 @@ describe('CodeIntrospectionService', () => {
console.log(x);
`;
const result = service.analyze(fileContent);
const result = service.getFunctionInputSchema(fileContent);
expect(result).toEqual([]);
});
@ -66,10 +66,10 @@ describe('CodeIntrospectionService', () => {
function func2(param2: number) {}
`;
expect(() => service.analyze(fileContent)).toThrow(
expect(() => service.getFunctionInputSchema(fileContent)).toThrow(
CodeIntrospectionException,
);
expect(() => service.analyze(fileContent)).toThrow(
expect(() => service.getFunctionInputSchema(fileContent)).toThrow(
'Only one function is allowed',
);
});
@ -80,10 +80,10 @@ describe('CodeIntrospectionService', () => {
const func2 = (param2: number) => {};
`;
expect(() => service.analyze(fileContent)).toThrow(
expect(() => service.getFunctionInputSchema(fileContent)).toThrow(
CodeIntrospectionException,
);
expect(() => service.analyze(fileContent)).toThrow(
expect(() => service.getFunctionInputSchema(fileContent)).toThrow(
'Only one arrow function is allowed',
);
});
@ -95,7 +95,7 @@ describe('CodeIntrospectionService', () => {
}
`;
const result = service.analyze(fileContent);
const result = service.getFunctionInputSchema(fileContent);
expect(result).toEqual([
{ name: 'param1', type: 'string[]' },

42
packages/twenty-server/src/modules/code-introspection/code-introspection.service.ts
View File

@ -8,16 +8,12 @@ import {
SyntaxKind,
} from 'ts-morph';
import { FunctionParameter } from 'src/engine/metadata-modules/serverless-function/dtos/function-parameter.dto';
import { generateFakeValue } from 'src/engine/utils/generate-fake-value';
import {
CodeIntrospectionException,
CodeIntrospectionExceptionCode,
} from 'src/modules/code-introspection/code-introspection.exception';
import { generateFakeValue } from 'src/engine/utils/generate-fake-value';
type FunctionParameter = {
name: string;
type: string;
};
@Injectable()
export class CodeIntrospectionService {
@ -27,7 +23,13 @@ export class CodeIntrospectionService {
this.project = new Project();
}
public analyze(
public generateInputData(fileContent: string, fileName = 'temp.ts') {
const parameters = this.getFunctionInputSchema(fileContent, fileName);
return this.generateFakeDataFromParams(parameters);
}
public getFunctionInputSchema(
fileContent: string,
fileName = 'temp.ts',
): FunctionParameter[] {
@ -38,7 +40,7 @@ export class CodeIntrospectionService {
const functionDeclarations = sourceFile.getFunctions();
if (functionDeclarations.length > 0) {
return this.analyzeFunctions(functionDeclarations);
return this.getFunctionParameters(functionDeclarations);
}
const arrowFunctions = sourceFile.getDescendantsOfKind(
@ -46,13 +48,13 @@ export class CodeIntrospectionService {
);
if (arrowFunctions.length > 0) {
return this.analyzeArrowFunctions(arrowFunctions);
return this.getArrowFunctionParameters(arrowFunctions);
}
return [];
}
private analyzeFunctions(
private getFunctionParameters(
functionDeclarations: FunctionDeclaration[],
): FunctionParameter[] {
if (functionDeclarations.length > 1) {
@ -67,7 +69,7 @@ export class CodeIntrospectionService {
return functionDeclaration.getParameters().map(this.buildFunctionParameter);
}
private analyzeArrowFunctions(
private getArrowFunctionParameters(
arrowFunctions: ArrowFunction[],
): FunctionParameter[] {
if (arrowFunctions.length > 1) {
@ -91,23 +93,13 @@ export class CodeIntrospectionService {
};
}
public generateInputData(fileContent: string, fileName = 'temp.ts') {
const parameters = this.analyze(fileContent, fileName);
return this.generateFakeDataFromParams(parameters);
}
private generateFakeDataFromParams(
params: FunctionParameter[],
): Record<string, any> {
const data: Record<string, any> = {};
return params.reduce((acc, param) => {
acc[param.name] = generateFakeValue(param.type);
params.forEach((param) => {
const type = param.type;
data[param.name] = generateFakeValue(type);
});
return data;
return acc;
}, {});
}
}

24
packages/twenty-server/src/modules/serverless/workflow-actions/code.workflow-action.ts
View File

@ -30,17 +30,21 @@ export class CodeWorkflowAction implements WorkflowAction {
);
}
const result =
await this.serverlessFunctionService.executeOneServerlessFunction(
workflowStepInput.serverlessFunctionId,
workspaceId,
{}, // TODO: input will be dynamically calculated from function input
);
try {
const result =
await this.serverlessFunctionService.executeOneServerlessFunction(
workflowStepInput.serverlessFunctionId,
workspaceId,
workflowStepInput.serverlessFunctionInput,
);
if (result.error) {
return { error: result.error };
if (result.error) {
return { error: result.error };
}
return { result: result.data || {} };
} catch (error) {
return { error: error.message };
}
return { result: result.data || {} };
}
}

8
packages/twenty-server/src/modules/workflow/workflow-builder/workflow-builder.module.ts
View File

@ -1,10 +1,10 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { WorkflowBuilderService } from 'src/modules/workflow/workflow-builder/workflow-builder.service';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module';
import { CodeIntrospectionModule } from 'src/modules/code-introspection/code-introspection.module';
import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity';
import { WorkflowBuilderWorkspaceService } from 'src/modules/workflow/workflow-builder/workflow-builder.workspace-service';
@Module({
imports: [
@ -12,7 +12,7 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
ServerlessFunctionModule,
CodeIntrospectionModule,
],
providers: [WorkflowBuilderService],
exports: [WorkflowBuilderService],
providers: [WorkflowBuilderWorkspaceService],
exports: [WorkflowBuilderWorkspaceService],
})
export class WorkflowBuilderModule {}

2
packages/twenty-server/src/modules/workflow/workflow-builder/workflow-builder.service.ts → packages/twenty-server/src/modules/workflow/workflow-builder/workflow-builder.workspace-service.ts
View File

@ -25,7 +25,7 @@ import { generateFakeObjectRecordEvent } from 'src/modules/workflow/workflow-bui
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
@Injectable()
export class WorkflowBuilderService {
export class WorkflowBuilderWorkspaceService {
constructor(
private readonly serverlessFunctionService: ServerlessFunctionService,
private readonly codeIntrospectionService: CodeIntrospectionService,

5
packages/twenty-server/src/modules/workflow/workflow-executor/types/workflow-step-settings.type.ts
View File

@ -1,4 +1,5 @@
export type OutputSchema = object;
export type InputSchema = object;
type BaseWorkflowStepSettings = {
input: object;
@ -16,7 +17,9 @@ type BaseWorkflowStepSettings = {
export type WorkflowCodeStepInput = {
serverlessFunctionId: string;
serverlessFunctionVersion: string;
payloadInput: object;
serverlessFunctionInput: {
[key: string]: any;
};
};
export type WorkflowCodeStepSettings = BaseWorkflowStepSettings & {

1
packages/twenty-server/src/modules/workflow/workflow-runner/workspace-services/workflow-run.workspace-service.ts
View File

@ -33,6 +33,7 @@ export class WorkflowRunWorkspaceService {
return (
await workflowRunRepository.save({
name: `Execution of ${workflowVersion.name}`,
workflowVersionId,
createdBy,
workflowId: workflowVersion.workflowId,