Poc lambda deployment duration (#10340)
closes https://github.com/twentyhq/core-team-issues/issues/436 ## Acheivements Improve aws lambda deployment time from ~10/15 secs to less that 1 sec ## Done - migrate with the new code executor architecture for local and lambda drivers - support old and new executor architecture to avoid breaking changes - first run is long, next runs are quick even if code step is updated ## Demo using `lambda` driver ### Before https://github.com/user-attachments/assets/7f7664b4-658f-4689-8949-ea2c31131252 ### After https://github.com/user-attachments/assets/d486c8e2-f8f8-4dbd-a801-c9901e440b29
This commit is contained in:
@ -0,0 +1,21 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
|
||||
export const handler = async (event) => {
|
||||
const mainPath = `/tmp/${v4()}.mjs`;
|
||||
|
||||
try {
|
||||
const { code, params } = event;
|
||||
|
||||
await fs.writeFile(mainPath, code, 'utf8');
|
||||
|
||||
process.env = {}
|
||||
|
||||
const mainFile = await import(mainPath);
|
||||
|
||||
return await mainFile.main(params);
|
||||
} finally {
|
||||
await fs.rm(mainPath, { force: true });
|
||||
}
|
||||
};
|
||||
@ -16,11 +16,7 @@ export type ServerlessExecuteResult = {
|
||||
|
||||
export interface ServerlessDriver {
|
||||
delete(serverlessFunction: ServerlessFunctionEntity): Promise<void>;
|
||||
build(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
version: string,
|
||||
): Promise<void>;
|
||||
publish(serverlessFunction: ServerlessFunctionEntity): Promise<string>;
|
||||
build(serverlessFunction: ServerlessFunctionEntity): Promise<void>;
|
||||
execute(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
payload: object,
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
import * as fs from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
|
||||
import ts, { transpileModule } from 'typescript';
|
||||
import {
|
||||
CreateFunctionCommandInput,
|
||||
CreateFunctionCommand,
|
||||
DeleteFunctionCommand,
|
||||
GetFunctionCommand,
|
||||
@ -13,18 +15,10 @@ import {
|
||||
ListLayerVersionsCommandInput,
|
||||
PublishLayerVersionCommand,
|
||||
PublishLayerVersionCommandInput,
|
||||
PublishVersionCommand,
|
||||
PublishVersionCommandInput,
|
||||
ResourceNotFoundException,
|
||||
UpdateFunctionCodeCommand,
|
||||
UpdateFunctionConfigurationCommand,
|
||||
UpdateFunctionConfigurationCommandInput,
|
||||
waitUntilFunctionUpdatedV2,
|
||||
} from '@aws-sdk/client-lambda';
|
||||
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
|
||||
import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
|
||||
import { UpdateFunctionCodeCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/UpdateFunctionCodeCommand';
|
||||
import dotenv from 'dotenv';
|
||||
import { isDefined } from 'twenty-shared';
|
||||
|
||||
import {
|
||||
@ -34,10 +28,6 @@ import {
|
||||
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
|
||||
import { ENV_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/env-file-name';
|
||||
import { OUTDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/outdir-folder';
|
||||
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
|
||||
import { compileTypescript } from 'src/engine/core-modules/serverless/drivers/utils/compile-typescript';
|
||||
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
|
||||
import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file';
|
||||
import {
|
||||
@ -54,9 +44,13 @@ import {
|
||||
ServerlessFunctionException,
|
||||
ServerlessFunctionExceptionCode,
|
||||
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
|
||||
import { copyExecutor } from 'src/engine/core-modules/serverless/drivers/utils/copy-executor';
|
||||
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
|
||||
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
|
||||
|
||||
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
|
||||
const CREDENTIALS_DURATION_IN_SECONDS = 10 * 60 * 60; // 10h
|
||||
const LAMBDA_EXECUTOR_DESCRIPTION = 'User script executor';
|
||||
|
||||
export interface LambdaDriverOptions extends LambdaClientConfig {
|
||||
fileStorageService: FileStorageService;
|
||||
@ -127,10 +121,12 @@ export class LambdaDriver implements ServerlessDriver {
|
||||
}
|
||||
|
||||
private async waitFunctionUpdates(
|
||||
serverlessFunctionId: string,
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
maxWaitTime: number = UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
|
||||
) {
|
||||
const waitParams = { FunctionName: serverlessFunctionId };
|
||||
const waitParams = {
|
||||
FunctionName: serverlessFunction.id,
|
||||
};
|
||||
|
||||
await waitUntilFunctionUpdatedV2(
|
||||
{ client: await this.getLambdaClient(), maxWaitTime },
|
||||
@ -192,29 +188,26 @@ export class LambdaDriver implements ServerlessDriver {
|
||||
return result.LayerVersionArn;
|
||||
}
|
||||
|
||||
private async checkFunctionExists(functionName: string): Promise<boolean> {
|
||||
private async getLambdaExecutor(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
) {
|
||||
try {
|
||||
const getFunctionCommand = new GetFunctionCommand({
|
||||
FunctionName: functionName,
|
||||
const getFunctionCommand: GetFunctionCommand = new GetFunctionCommand({
|
||||
FunctionName: serverlessFunction.id,
|
||||
});
|
||||
|
||||
await (await this.getLambdaClient()).send(getFunctionCommand);
|
||||
|
||||
return true;
|
||||
return await (await this.getLambdaClient()).send(getFunctionCommand);
|
||||
} catch (error) {
|
||||
if (error instanceof ResourceNotFoundException) {
|
||||
return false;
|
||||
if (!(error instanceof ResourceNotFoundException)) {
|
||||
throw error;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async delete(serverlessFunction: ServerlessFunctionEntity) {
|
||||
const functionExists = await this.checkFunctionExists(
|
||||
serverlessFunction.id,
|
||||
);
|
||||
const lambdaExecutor = await this.getLambdaExecutor(serverlessFunction);
|
||||
|
||||
if (functionExists) {
|
||||
if (isDefined(lambdaExecutor)) {
|
||||
const deleteFunctionCommand = new DeleteFunctionCommand({
|
||||
FunctionName: serverlessFunction.id,
|
||||
});
|
||||
@ -223,162 +216,92 @@ export class LambdaDriver implements ServerlessDriver {
|
||||
}
|
||||
}
|
||||
|
||||
private getInMemoryServerlessFunctionFolderPath = (
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
version: string,
|
||||
) => {
|
||||
return join(SERVERLESS_TMPDIR_FOLDER, serverlessFunction.id, version);
|
||||
};
|
||||
async build(serverlessFunction: ServerlessFunctionEntity) {
|
||||
const lambdaExecutor = await this.getLambdaExecutor(serverlessFunction);
|
||||
|
||||
async build(serverlessFunction: ServerlessFunctionEntity, version: 'draft') {
|
||||
if (version !== 'draft') {
|
||||
throw new Error("We can only build 'draft' version with lambda driver");
|
||||
if (isDefined(lambdaExecutor)) {
|
||||
if (
|
||||
lambdaExecutor.Configuration?.Description ===
|
||||
LAMBDA_EXECUTOR_DESCRIPTION
|
||||
) {
|
||||
return;
|
||||
}
|
||||
await this.delete(serverlessFunction);
|
||||
}
|
||||
|
||||
const inMemoryServerlessFunctionFolderPath =
|
||||
this.getInMemoryServerlessFunctionFolderPath(serverlessFunction, version);
|
||||
|
||||
const folderPath = getServerlessFolder({
|
||||
serverlessFunction,
|
||||
version,
|
||||
});
|
||||
|
||||
await this.fileStorageService.download({
|
||||
from: { folderPath },
|
||||
to: { folderPath: inMemoryServerlessFunctionFolderPath },
|
||||
});
|
||||
|
||||
compileTypescript(inMemoryServerlessFunctionFolderPath);
|
||||
|
||||
const lambdaZipPath = join(
|
||||
inMemoryServerlessFunctionFolderPath,
|
||||
'lambda.zip',
|
||||
const layerArn = await this.createLayerIfNotExists(
|
||||
serverlessFunction.layerVersion,
|
||||
);
|
||||
|
||||
await createZipFile(
|
||||
join(inMemoryServerlessFunctionFolderPath, OUTDIR_FOLDER),
|
||||
lambdaZipPath,
|
||||
);
|
||||
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
|
||||
|
||||
const envFileContent = await fs.readFile(
|
||||
join(inMemoryServerlessFunctionFolderPath, ENV_FILE_NAME),
|
||||
);
|
||||
const { sourceTemporaryDir, lambdaZipPath } =
|
||||
await lambdaBuildDirectoryManager.init();
|
||||
|
||||
const envVariables = dotenv.parse(envFileContent);
|
||||
await copyExecutor(sourceTemporaryDir);
|
||||
|
||||
const functionExists = await this.checkFunctionExists(
|
||||
serverlessFunction.id,
|
||||
);
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
|
||||
if (!functionExists) {
|
||||
const layerArn = await this.createLayerIfNotExists(
|
||||
serverlessFunction.layerVersion,
|
||||
);
|
||||
|
||||
const params: CreateFunctionCommandInput = {
|
||||
Code: {
|
||||
ZipFile: await fs.readFile(lambdaZipPath),
|
||||
},
|
||||
FunctionName: serverlessFunction.id,
|
||||
Handler: 'src/index.main',
|
||||
Layers: [layerArn],
|
||||
Environment: {
|
||||
Variables: envVariables,
|
||||
},
|
||||
Role: this.options.lambdaRole,
|
||||
Runtime: serverlessFunction.runtime,
|
||||
Description: 'Lambda function to run user script',
|
||||
Timeout: serverlessFunction.timeoutSeconds,
|
||||
};
|
||||
|
||||
const command = new CreateFunctionCommand(params);
|
||||
|
||||
await (await this.getLambdaClient()).send(command);
|
||||
} else {
|
||||
const updateCodeParams: UpdateFunctionCodeCommandInput = {
|
||||
const params: CreateFunctionCommandInput = {
|
||||
Code: {
|
||||
ZipFile: await fs.readFile(lambdaZipPath),
|
||||
FunctionName: serverlessFunction.id,
|
||||
};
|
||||
|
||||
const updateCodeCommand = new UpdateFunctionCodeCommand(updateCodeParams);
|
||||
|
||||
await (await this.getLambdaClient()).send(updateCodeCommand);
|
||||
|
||||
const updateConfigurationParams: UpdateFunctionConfigurationCommandInput =
|
||||
{
|
||||
Environment: {
|
||||
Variables: envVariables,
|
||||
},
|
||||
FunctionName: serverlessFunction.id,
|
||||
Timeout: serverlessFunction.timeoutSeconds,
|
||||
};
|
||||
|
||||
const updateConfigurationCommand = new UpdateFunctionConfigurationCommand(
|
||||
updateConfigurationParams,
|
||||
);
|
||||
|
||||
await this.waitFunctionUpdates(serverlessFunction.id);
|
||||
|
||||
await (await this.getLambdaClient()).send(updateConfigurationCommand);
|
||||
}
|
||||
|
||||
await this.waitFunctionUpdates(serverlessFunction.id);
|
||||
}
|
||||
|
||||
async publish(serverlessFunction: ServerlessFunctionEntity) {
|
||||
await this.build(serverlessFunction, 'draft');
|
||||
const params: PublishVersionCommandInput = {
|
||||
},
|
||||
FunctionName: serverlessFunction.id,
|
||||
Layers: [layerArn],
|
||||
Handler: 'index.handler',
|
||||
Role: this.options.lambdaRole,
|
||||
Runtime: serverlessFunction.runtime,
|
||||
Description: LAMBDA_EXECUTOR_DESCRIPTION,
|
||||
Timeout: serverlessFunction.timeoutSeconds,
|
||||
};
|
||||
|
||||
const command = new PublishVersionCommand(params);
|
||||
const command = new CreateFunctionCommand(params);
|
||||
|
||||
const result = await (await this.getLambdaClient()).send(command);
|
||||
const newVersion = result.Version;
|
||||
await (await this.getLambdaClient()).send(command);
|
||||
|
||||
if (!newVersion) {
|
||||
throw new Error('New published version is undefined');
|
||||
}
|
||||
|
||||
const draftFolderPath = getServerlessFolder({
|
||||
serverlessFunction: serverlessFunction,
|
||||
version: 'draft',
|
||||
});
|
||||
const newFolderPath = getServerlessFolder({
|
||||
serverlessFunction: serverlessFunction,
|
||||
version: newVersion,
|
||||
});
|
||||
|
||||
await this.fileStorageService.copy({
|
||||
from: { folderPath: draftFolderPath },
|
||||
to: { folderPath: newFolderPath },
|
||||
});
|
||||
|
||||
return newVersion;
|
||||
await lambdaBuildDirectoryManager.clean();
|
||||
}
|
||||
|
||||
async execute(
|
||||
functionToExecute: ServerlessFunctionEntity,
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
payload: object,
|
||||
version: string,
|
||||
): Promise<ServerlessExecuteResult> {
|
||||
const computedVersion =
|
||||
version === 'latest' ? functionToExecute.latestVersion : version;
|
||||
|
||||
const functionName =
|
||||
computedVersion === 'draft'
|
||||
? functionToExecute.id
|
||||
: `${functionToExecute.id}:${computedVersion}`;
|
||||
|
||||
if (version === 'draft') {
|
||||
await this.waitFunctionUpdates(functionToExecute.id);
|
||||
}
|
||||
await this.build(serverlessFunction);
|
||||
await this.waitFunctionUpdates(serverlessFunction);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
const computedVersion =
|
||||
version === 'latest' ? serverlessFunction.latestVersion : version;
|
||||
|
||||
const folderPath = getServerlessFolder({
|
||||
serverlessFunction,
|
||||
version: computedVersion,
|
||||
});
|
||||
|
||||
const tsCodeStream = await this.fileStorageService.read({
|
||||
folderPath: join(folderPath, 'src'),
|
||||
filename: INDEX_FILE_NAME,
|
||||
});
|
||||
|
||||
const tsCode = await readFileContent(tsCodeStream);
|
||||
|
||||
const compiledCode = transpileModule(tsCode, {
|
||||
compilerOptions: {
|
||||
module: ts.ModuleKind.ESNext,
|
||||
target: ts.ScriptTarget.ES2017,
|
||||
},
|
||||
}).outputText;
|
||||
|
||||
const executorPayload = {
|
||||
params: payload,
|
||||
code: compiledCode,
|
||||
};
|
||||
|
||||
const params: InvokeCommandInput = {
|
||||
FunctionName: functionName,
|
||||
Payload: JSON.stringify(payload),
|
||||
FunctionName: serverlessFunction.id,
|
||||
Payload: JSON.stringify(executorPayload),
|
||||
};
|
||||
|
||||
const command = new InvokeCommand(params);
|
||||
|
||||
@ -1,27 +1,23 @@
|
||||
import { fork } from 'child_process';
|
||||
import { promises as fs } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
import dotenv from 'dotenv';
|
||||
import ts, { transpileModule } from 'typescript';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import {
|
||||
ServerlessDriver,
|
||||
ServerlessExecuteError,
|
||||
ServerlessExecuteResult,
|
||||
} from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
|
||||
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
|
||||
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
|
||||
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
|
||||
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
|
||||
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
|
||||
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
|
||||
import { compileTypescript } from 'src/engine/core-modules/serverless/drivers/utils/compile-typescript';
|
||||
import { OUTDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/outdir-folder';
|
||||
import { ENV_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/env-file-name';
|
||||
|
||||
const LISTENER_FILE_NAME = 'listener.js';
|
||||
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
|
||||
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
|
||||
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
|
||||
|
||||
export interface LocalDriverOptions {
|
||||
fileStorageService: FileStorageService;
|
||||
@ -34,13 +30,6 @@ export class LocalDriver implements ServerlessDriver {
|
||||
this.fileStorageService = options.fileStorageService;
|
||||
}
|
||||
|
||||
private getInMemoryServerlessFunctionFolderPath = (
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
version: string,
|
||||
) => {
|
||||
return join(SERVERLESS_TMPDIR_FOLDER, serverlessFunction.id, version);
|
||||
};
|
||||
|
||||
private getInMemoryLayerFolderPath = (version: number) => {
|
||||
return join(SERVERLESS_TMPDIR_FOLDER, COMMON_LAYER_NAME, `${version}`);
|
||||
};
|
||||
@ -58,107 +47,29 @@ export class LocalDriver implements ServerlessDriver {
|
||||
|
||||
async delete() {}
|
||||
|
||||
async build(serverlessFunction: ServerlessFunctionEntity, version: string) {
|
||||
const computedVersion =
|
||||
version === 'latest' ? serverlessFunction.latestVersion : version;
|
||||
|
||||
async build(serverlessFunction: ServerlessFunctionEntity) {
|
||||
await this.createLayerIfNotExists(serverlessFunction.layerVersion);
|
||||
|
||||
const inMemoryServerlessFunctionFolderPath =
|
||||
this.getInMemoryServerlessFunctionFolderPath(
|
||||
serverlessFunction,
|
||||
computedVersion,
|
||||
);
|
||||
|
||||
const folderPath = getServerlessFolder({
|
||||
serverlessFunction,
|
||||
version,
|
||||
});
|
||||
|
||||
await this.fileStorageService.download({
|
||||
from: { folderPath },
|
||||
to: { folderPath: inMemoryServerlessFunctionFolderPath },
|
||||
});
|
||||
|
||||
compileTypescript(inMemoryServerlessFunctionFolderPath);
|
||||
|
||||
const envFileContent = await fs.readFile(
|
||||
join(inMemoryServerlessFunctionFolderPath, ENV_FILE_NAME),
|
||||
);
|
||||
|
||||
const envVariables = dotenv.parse(envFileContent);
|
||||
|
||||
const listener = `
|
||||
const index_1 = require("./src/index");
|
||||
|
||||
process.env = ${JSON.stringify(envVariables)}
|
||||
|
||||
process.on('message', async (message) => {
|
||||
const { params } = message;
|
||||
try {
|
||||
const result = await index_1.main(params);
|
||||
process.send(result);
|
||||
} catch (error) {
|
||||
process.send({
|
||||
errorType: error.name,
|
||||
errorMessage: error.message,
|
||||
stackTrace: error.stack.split('\\n').filter((line) => line.trim() !== ''),
|
||||
});
|
||||
}
|
||||
});
|
||||
`;
|
||||
|
||||
await fs.writeFile(
|
||||
join(
|
||||
inMemoryServerlessFunctionFolderPath,
|
||||
OUTDIR_FOLDER,
|
||||
LISTENER_FILE_NAME,
|
||||
),
|
||||
listener,
|
||||
);
|
||||
|
||||
try {
|
||||
await fs.symlink(
|
||||
join(
|
||||
this.getInMemoryLayerFolderPath(serverlessFunction.layerVersion),
|
||||
'node_modules',
|
||||
),
|
||||
join(
|
||||
inMemoryServerlessFunctionFolderPath,
|
||||
OUTDIR_FOLDER,
|
||||
'node_modules',
|
||||
),
|
||||
'dir',
|
||||
);
|
||||
} catch (err) {
|
||||
if (err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async publish(serverlessFunction: ServerlessFunctionEntity) {
|
||||
const newVersion = serverlessFunction.latestVersion
|
||||
? `${parseInt(serverlessFunction.latestVersion, 10) + 1}`
|
||||
: '1';
|
||||
private async executeWithTimeout<T>(
|
||||
fn: () => Promise<T>,
|
||||
timeoutMs: number,
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
reject(new Error(`Task timed out after ${timeoutMs / 1_000} seconds`));
|
||||
}, timeoutMs);
|
||||
|
||||
const draftFolderPath = getServerlessFolder({
|
||||
serverlessFunction: serverlessFunction,
|
||||
version: 'draft',
|
||||
fn()
|
||||
.then((result) => {
|
||||
clearTimeout(timer);
|
||||
resolve(result);
|
||||
})
|
||||
.catch((err) => {
|
||||
clearTimeout(timer);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
const newFolderPath = getServerlessFolder({
|
||||
serverlessFunction: serverlessFunction,
|
||||
version: newVersion,
|
||||
});
|
||||
|
||||
await this.fileStorageService.copy({
|
||||
from: { folderPath: draftFolderPath },
|
||||
to: { folderPath: newFolderPath },
|
||||
});
|
||||
|
||||
await this.build(serverlessFunction, newVersion);
|
||||
|
||||
return newVersion;
|
||||
}
|
||||
|
||||
async execute(
|
||||
@ -166,100 +77,73 @@ export class LocalDriver implements ServerlessDriver {
|
||||
payload: object,
|
||||
version: string,
|
||||
): Promise<ServerlessExecuteResult> {
|
||||
await this.build(serverlessFunction);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
const computedVersion =
|
||||
version === 'latest' ? serverlessFunction.latestVersion : version;
|
||||
|
||||
const listenerFile = join(
|
||||
this.getInMemoryServerlessFunctionFolderPath(
|
||||
serverlessFunction,
|
||||
computedVersion,
|
||||
),
|
||||
OUTDIR_FOLDER,
|
||||
LISTENER_FILE_NAME,
|
||||
const folderPath = getServerlessFolder({
|
||||
serverlessFunction,
|
||||
version: computedVersion,
|
||||
});
|
||||
|
||||
const tsCodeStream = await this.fileStorageService.read({
|
||||
folderPath: join(folderPath, 'src'),
|
||||
filename: INDEX_FILE_NAME,
|
||||
});
|
||||
|
||||
const tsCode = await readFileContent(tsCodeStream);
|
||||
|
||||
const compiledCode = transpileModule(tsCode, {
|
||||
compilerOptions: {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2017,
|
||||
},
|
||||
}).outputText;
|
||||
|
||||
const compiledCodeFolderPath = join(
|
||||
SERVERLESS_TMPDIR_FOLDER,
|
||||
`compiled-code-${v4()}`,
|
||||
);
|
||||
|
||||
const compiledCodeFilePath = join(compiledCodeFolderPath, 'main.js');
|
||||
|
||||
await fs.mkdir(compiledCodeFolderPath, { recursive: true });
|
||||
|
||||
await fs.writeFile(compiledCodeFilePath, compiledCode, 'utf8');
|
||||
|
||||
try {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const child = fork(listenerFile, { silent: true });
|
||||
await fs.symlink(
|
||||
join(
|
||||
this.getInMemoryLayerFolderPath(serverlessFunction.layerVersion),
|
||||
'node_modules',
|
||||
),
|
||||
join(compiledCodeFolderPath, 'node_modules'),
|
||||
'dir',
|
||||
);
|
||||
} catch (err) {
|
||||
if (err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const timeoutMs = serverlessFunction.timeoutSeconds * 1_000;
|
||||
try {
|
||||
const mainFile = await import(compiledCodeFilePath);
|
||||
|
||||
const timeoutHandler = setTimeout(() => {
|
||||
child.kill();
|
||||
const duration = Date.now() - startTime;
|
||||
const result = await this.executeWithTimeout<object | null>(
|
||||
() => mainFile.main(payload),
|
||||
serverlessFunction.timeoutSeconds * 1_000,
|
||||
);
|
||||
|
||||
reject(new Error(`Task timed out after ${duration / 1_000} seconds`));
|
||||
}, timeoutMs);
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
child.on('message', (message: object | ServerlessExecuteError) => {
|
||||
clearTimeout(timeoutHandler);
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
if ('errorType' in message) {
|
||||
resolve({
|
||||
data: null,
|
||||
duration,
|
||||
error: message,
|
||||
status: ServerlessFunctionExecutionStatus.ERROR,
|
||||
});
|
||||
} else {
|
||||
resolve({
|
||||
data: message,
|
||||
duration,
|
||||
status: ServerlessFunctionExecutionStatus.SUCCESS,
|
||||
});
|
||||
}
|
||||
child.kill();
|
||||
});
|
||||
|
||||
child.stderr?.on('data', (data) => {
|
||||
clearTimeout(timeoutHandler);
|
||||
const stackTrace = data
|
||||
.toString()
|
||||
.split('\n')
|
||||
.filter((line: string) => line.trim() !== '');
|
||||
const errorTrace = stackTrace.filter((line: string) =>
|
||||
line.includes('Error: '),
|
||||
)?.[0];
|
||||
|
||||
let errorType = 'Unknown';
|
||||
let errorMessage = '';
|
||||
|
||||
if (errorTrace) {
|
||||
errorType = errorTrace.split(':')[0];
|
||||
errorMessage = errorTrace.split(': ')[1];
|
||||
}
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
resolve({
|
||||
data: null,
|
||||
duration,
|
||||
status: ServerlessFunctionExecutionStatus.ERROR,
|
||||
error: {
|
||||
errorType,
|
||||
errorMessage,
|
||||
stackTrace: stackTrace,
|
||||
},
|
||||
});
|
||||
child.kill();
|
||||
});
|
||||
|
||||
child.on('error', (error) => {
|
||||
clearTimeout(timeoutHandler);
|
||||
reject(error);
|
||||
child.kill();
|
||||
});
|
||||
|
||||
child.on('exit', (code) => {
|
||||
clearTimeout(timeoutHandler);
|
||||
if (code && code !== 0) {
|
||||
reject(new Error(`Child process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
|
||||
child.send({ params: payload });
|
||||
});
|
||||
return {
|
||||
data: result,
|
||||
duration,
|
||||
status: ServerlessFunctionExecutionStatus.SUCCESS,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
data: null,
|
||||
@ -271,6 +155,8 @@ export class LocalDriver implements ServerlessDriver {
|
||||
},
|
||||
status: ServerlessFunctionExecutionStatus.ERROR,
|
||||
};
|
||||
} finally {
|
||||
await fs.rm(compiledCodeFolderPath, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
import { join } from 'path';
|
||||
|
||||
import ts, { createProgram } from 'typescript';
|
||||
|
||||
import { OUTDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/outdir-folder';
|
||||
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
|
||||
|
||||
export const compileTypescript = (folderPath: string) => {
|
||||
const options: ts.CompilerOptions = {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2017,
|
||||
moduleResolution: ts.ModuleResolutionKind.Node10,
|
||||
esModuleInterop: true,
|
||||
resolveJsonModule: true,
|
||||
allowSyntheticDefaultImports: true,
|
||||
outDir: join(folderPath, OUTDIR_FOLDER, 'src'),
|
||||
types: ['node'],
|
||||
};
|
||||
|
||||
createProgram([join(folderPath, 'src', INDEX_FILE_NAME)], options).emit();
|
||||
};
|
||||
@ -0,0 +1,12 @@
|
||||
import { promises as fs } from 'fs';
|
||||
|
||||
import { getExecutorFilePath } from 'src/engine/core-modules/serverless/drivers/utils/get-executor-file-path';
|
||||
|
||||
export const copyExecutor = async (buildDirectory: string) => {
|
||||
await fs.mkdir(buildDirectory, {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.cp(getExecutorFilePath(), buildDirectory, {
|
||||
recursive: true,
|
||||
});
|
||||
};
|
||||
@ -0,0 +1,12 @@
|
||||
import path from 'path';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
|
||||
export const getExecutorFilePath = (): string => {
|
||||
const baseTypescriptProjectPath = path.join(
|
||||
ASSET_PATH,
|
||||
`engine/core-modules/serverless/drivers/constants/executor`,
|
||||
);
|
||||
|
||||
return path.resolve(__dirname, baseTypescriptProjectPath);
|
||||
};
|
||||
@ -1,4 +1,4 @@
|
||||
import path, { join } from 'path';
|
||||
import path from 'path';
|
||||
|
||||
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
@ -8,10 +8,10 @@ export const getLayerDependenciesDirName = (
|
||||
): string => {
|
||||
const formattedVersion = version === 'latest' ? LAST_LAYER_VERSION : version;
|
||||
|
||||
const baseTypescriptProjectPath = join(
|
||||
const baseTypescriptProjectPath = path.join(
|
||||
ASSET_PATH,
|
||||
`engine/core-modules/serverless/drivers/layers/${formattedVersion}`,
|
||||
);
|
||||
|
||||
return path.resolve(baseTypescriptProjectPath);
|
||||
return path.resolve(__dirname, baseTypescriptProjectPath);
|
||||
};
|
||||
|
||||
@ -16,15 +16,8 @@ export class ServerlessService implements ServerlessDriver {
|
||||
return this.driver.delete(serverlessFunction);
|
||||
}
|
||||
|
||||
async build(
|
||||
serverlessFunction: ServerlessFunctionEntity,
|
||||
version: string,
|
||||
): Promise<void> {
|
||||
return this.driver.build(serverlessFunction, version);
|
||||
}
|
||||
|
||||
async publish(serverlessFunction: ServerlessFunctionEntity): Promise<string> {
|
||||
return this.driver.publish(serverlessFunction);
|
||||
async build(serverlessFunction: ServerlessFunctionEntity): Promise<void> {
|
||||
return this.driver.build(serverlessFunction);
|
||||
}
|
||||
|
||||
async execute(
|
||||
|
||||
Reference in New Issue
Block a user