6653 serverless functions store and use environment variables in serverless function scripts (#7390)

![image](https://github.com/user-attachments/assets/a15bd4c1-3db4-4466-b748-06bdf3874354)

![image](https://github.com/user-attachments/assets/71242dfb-956b-43ed-9704-87cb0dfbc98d)
This commit is contained in:
martmull
2024-10-03 13:56:17 +02:00
committed by GitHub
parent 3cd24d542b
commit 62fe1d0e88
39 changed files with 815 additions and 513 deletions

View File

@ -1,25 +0,0 @@
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { SOURCE_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/source-file-name';
import { compileTypescript } from 'src/engine/core-modules/serverless/drivers/utils/compile-typescript';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
export class BaseServerlessDriver {
async getCompiledCode(
serverlessFunction: ServerlessFunctionEntity,
fileStorageService: FileStorageService,
) {
const folderPath = getServerlessFolder({
serverlessFunction,
version: 'draft',
});
const fileStream = await fileStorageService.read({
folderPath,
filename: SOURCE_FILE_NAME,
});
const typescriptCode = await readFileContent(fileStream);
return compileTypescript(typescriptCode);
}
}

View File

@ -0,0 +1 @@
!base-typescript-project/**/.env

View File

@ -0,0 +1,2 @@
# Add your environment variables here.
# Access them in your serverless function code using process.env.VARIABLE

View File

@ -0,0 +1,7 @@
export const handler = async (
event: object,
context: object,
): Promise<object> => {
// Your code here
return {};
};

View File

@ -0,0 +1 @@
export const ENV_FILE_NAME = '.env';

View File

@ -0,0 +1 @@
export const INDEX_FILE_NAME = 'index.ts';

View File

@ -0,0 +1 @@
export const OUTDIR_FOLDER = 'dist';

View File

@ -1 +0,0 @@
export const SOURCE_FILE_NAME = 'source.ts';

View File

@ -1,6 +1,7 @@
import * as fs from 'fs/promises';
import { join } from 'path';
import dotenv from 'dotenv';
import {
CreateFunctionCommand,
DeleteFunctionCommand,
@ -18,6 +19,8 @@ import {
waitUntilFunctionUpdatedV2,
ListLayerVersionsCommandInput,
ListLayerVersionsCommand,
UpdateFunctionConfigurationCommand,
UpdateFunctionConfigurationCommandInput,
} from '@aws-sdk/client-lambda';
import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
import { UpdateFunctionCodeCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/UpdateFunctionCodeCommand';
@ -36,7 +39,6 @@ import {
NODE_LAYER_SUBFOLDER,
} from 'src/engine/core-modules/serverless/drivers/utils/lambda-build-directory-manager';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { BaseServerlessDriver } from 'src/engine/core-modules/serverless/drivers/base-serverless.driver';
import { createZipFile } from 'src/engine/core-modules/serverless/drivers/utils/create-zip-file';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import {
@ -46,6 +48,11 @@ import {
import { isDefined } from 'src/utils/is-defined';
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
import { compileTypescript } from 'src/engine/core-modules/serverless/drivers/utils/compile-typescript';
import { ENV_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/env-file-name';
import { OUTDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/outdir-folder';
export interface LambdaDriverOptions extends LambdaClientConfig {
fileStorageService: FileStorageService;
@ -53,16 +60,12 @@ export interface LambdaDriverOptions extends LambdaClientConfig {
role: string;
}
export class LambdaDriver
extends BaseServerlessDriver
implements ServerlessDriver
{
export class LambdaDriver implements ServerlessDriver {
private readonly lambdaClient: Lambda;
private readonly lambdaRole: string;
private readonly fileStorageService: FileStorageService;
constructor(options: LambdaDriverOptions) {
super();
const { region, role, ...lambdaOptions } = options;
this.lambdaClient = new Lambda({ ...lambdaOptions, region });
@ -165,24 +168,50 @@ export class LambdaDriver
}
}
async build(serverlessFunction: ServerlessFunctionEntity) {
const javascriptCode = await this.getCompiledCode(
private getInMemoryServerlessFunctionFolderPath = (
serverlessFunction: ServerlessFunctionEntity,
version: string,
) => {
return join(SERVERLESS_TMPDIR_FOLDER, serverlessFunction.id, version);
};
async build(serverlessFunction: ServerlessFunctionEntity, version: string) {
const computedVersion =
version === 'latest' ? serverlessFunction.latestVersion : version;
const inMemoryServerlessFunctionFolderPath =
this.getInMemoryServerlessFunctionFolderPath(
serverlessFunction,
computedVersion,
);
const folderPath = getServerlessFolder({
serverlessFunction,
this.fileStorageService,
version,
});
await this.fileStorageService.download({
from: { folderPath },
to: { folderPath: inMemoryServerlessFunctionFolderPath },
});
compileTypescript(inMemoryServerlessFunctionFolderPath);
const lambdaZipPath = join(
inMemoryServerlessFunctionFolderPath,
'lambda.zip',
);
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const {
sourceTemporaryDir,
await createZipFile(
join(inMemoryServerlessFunctionFolderPath, OUTDIR_FOLDER),
lambdaZipPath,
javascriptFilePath,
lambdaHandler,
} = await lambdaBuildDirectoryManager.init();
);
await fs.writeFile(javascriptFilePath, javascriptCode);
const envFileContent = await fs.readFile(
join(inMemoryServerlessFunctionFolderPath, ENV_FILE_NAME),
);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const envVariables = dotenv.parse(envFileContent);
const functionExists = await this.checkFunctionExists(
serverlessFunction.id,
@ -198,8 +227,11 @@ export class LambdaDriver
ZipFile: await fs.readFile(lambdaZipPath),
},
FunctionName: serverlessFunction.id,
Handler: lambdaHandler,
Handler: 'src/index.handler',
Layers: [layerArn],
Environment: {
Variables: envVariables,
},
Role: this.lambdaRole,
Runtime: serverlessFunction.runtime,
Description: 'Lambda function to run user script',
@ -210,23 +242,37 @@ export class LambdaDriver
await this.lambdaClient.send(command);
} else {
const params: UpdateFunctionCodeCommandInput = {
const updateCodeParams: UpdateFunctionCodeCommandInput = {
ZipFile: await fs.readFile(lambdaZipPath),
FunctionName: serverlessFunction.id,
};
const command = new UpdateFunctionCodeCommand(params);
const updateCodeCommand = new UpdateFunctionCodeCommand(updateCodeParams);
await this.lambdaClient.send(command);
await this.lambdaClient.send(updateCodeCommand);
const updateConfigurationParams: UpdateFunctionConfigurationCommandInput =
{
Environment: {
Variables: envVariables,
},
FunctionName: serverlessFunction.id,
};
const updateConfigurationCommand = new UpdateFunctionConfigurationCommand(
updateConfigurationParams,
);
await this.waitFunctionUpdates(serverlessFunction.id, 10);
await this.lambdaClient.send(updateConfigurationCommand);
}
await this.waitFunctionUpdates(serverlessFunction.id, 10);
await lambdaBuildDirectoryManager.clean();
}
async publish(serverlessFunction: ServerlessFunctionEntity) {
await this.build(serverlessFunction);
await this.build(serverlessFunction, 'draft');
const params: PublishVersionCommandInput = {
FunctionName: serverlessFunction.id,
};
@ -240,6 +286,20 @@ export class LambdaDriver
throw new Error('New published version is undefined');
}
const draftFolderPath = getServerlessFolder({
serverlessFunction: serverlessFunction,
version: 'draft',
});
const newFolderPath = getServerlessFolder({
serverlessFunction: serverlessFunction,
version: newVersion,
});
await this.fileStorageService.copy({
from: { folderPath: draftFolderPath },
to: { folderPath: newFolderPath },
});
return newVersion;
}

View File

@ -1,10 +1,9 @@
import { fork } from 'child_process';
import { promises as fs, existsSync } from 'fs';
import { promises as fs } from 'fs';
import { join } from 'path';
import { v4 } from 'uuid';
import dotenv from 'dotenv';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import {
ServerlessDriver,
ServerlessExecuteError,
@ -12,35 +11,36 @@ import {
} from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { BaseServerlessDriver } from 'src/engine/core-modules/serverless/drivers/base-serverless.driver';
import { BUILD_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/build-file-name';
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { COMMON_LAYER_NAME } from 'src/engine/core-modules/serverless/drivers/constants/common-layer-name';
import { copyAndBuildDependencies } from 'src/engine/core-modules/serverless/drivers/utils/copy-and-build-dependencies';
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/serverless-tmpdir-folder';
import { compileTypescript } from 'src/engine/core-modules/serverless/drivers/utils/compile-typescript';
import { OUTDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/outdir-folder';
import { ENV_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/env-file-name';
const LISTENER_FILE_NAME = 'listener.js';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
}
export class LocalDriver
extends BaseServerlessDriver
implements ServerlessDriver
{
export class LocalDriver implements ServerlessDriver {
private readonly fileStorageService: FileStorageService;
constructor(options: LocalDriverOptions) {
super();
this.fileStorageService = options.fileStorageService;
}
private getInMemoryServerlessFunctionFolderPath = (
serverlessFunction: ServerlessFunctionEntity,
version: string,
) => {
return join(SERVERLESS_TMPDIR_FOLDER, serverlessFunction.id, version);
};
private getInMemoryLayerFolderPath = (version: number) => {
return join(SERVERLESS_TMPDIR_FOLDER, COMMON_LAYER_NAME, `${version}`);
};
@ -49,88 +49,54 @@ export class LocalDriver
const inMemoryLastVersionLayerFolderPath =
this.getInMemoryLayerFolderPath(version);
if (existsSync(inMemoryLastVersionLayerFolderPath)) {
return;
try {
await fs.access(inMemoryLastVersionLayerFolderPath);
} catch (e) {
await copyAndBuildDependencies(inMemoryLastVersionLayerFolderPath);
}
await copyAndBuildDependencies(inMemoryLastVersionLayerFolderPath);
}
async delete() {}
async build(serverlessFunction: ServerlessFunctionEntity) {
await this.createLayerIfNotExists(serverlessFunction.layerVersion);
const javascriptCode = await this.getCompiledCode(
serverlessFunction,
this.fileStorageService,
);
async build(serverlessFunction: ServerlessFunctionEntity, version: string) {
const computedVersion =
version === 'latest' ? serverlessFunction.latestVersion : version;
const draftFolderPath = getServerlessFolder({
serverlessFunction,
version: 'draft',
});
await this.fileStorageService.write({
file: javascriptCode,
name: BUILD_FILE_NAME,
mimeType: undefined,
folder: draftFolderPath,
});
}
async publish(serverlessFunction: ServerlessFunctionEntity) {
await this.build(serverlessFunction);
return serverlessFunction.latestVersion
? `${parseInt(serverlessFunction.latestVersion, 10) + 1}`
: '1';
}
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object,
version: string,
): Promise<ServerlessExecuteResult> {
await this.createLayerIfNotExists(serverlessFunction.layerVersion);
const startTime = Date.now();
let fileContent = '';
const inMemoryServerlessFunctionFolderPath =
this.getInMemoryServerlessFunctionFolderPath(
serverlessFunction,
computedVersion,
);
try {
const fileStream = await this.fileStorageService.read({
folderPath: getServerlessFolder({
serverlessFunction,
version,
}),
filename: BUILD_FILE_NAME,
});
const folderPath = getServerlessFolder({
serverlessFunction,
version,
});
fileContent = await readFileContent(fileStream);
} catch (error) {
if (error.code === FileStorageExceptionCode.FILE_NOT_FOUND) {
throw new ServerlessFunctionException(
`Function Version '${version}' does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
throw error;
}
await this.fileStorageService.download({
from: { folderPath },
to: { folderPath: inMemoryServerlessFunctionFolderPath },
});
const tmpFolderPath = join(SERVERLESS_TMPDIR_FOLDER, v4());
compileTypescript(inMemoryServerlessFunctionFolderPath);
const tmpFilePath = join(tmpFolderPath, 'index.js');
await fs.symlink(
this.getInMemoryLayerFolderPath(serverlessFunction.layerVersion),
tmpFolderPath,
'dir',
const envFileContent = await fs.readFile(
join(inMemoryServerlessFunctionFolderPath, ENV_FILE_NAME),
);
const modifiedContent = `
const envVariables = dotenv.parse(envFileContent);
const listener = `
const index_1 = require("./src/index");
process.env = ${JSON.stringify(envVariables)}
process.on('message', async (message) => {
const { event, context } = message;
try {
const result = await handler(event, context);
const result = await index_1.handler(event, context);
process.send(result);
} catch (error) {
process.send({
@ -140,82 +106,158 @@ export class LocalDriver
});
}
});
${fileContent}
`;
await fs.writeFile(tmpFilePath, modifiedContent);
await fs.writeFile(
join(
inMemoryServerlessFunctionFolderPath,
OUTDIR_FOLDER,
LISTENER_FILE_NAME,
),
listener,
);
return await new Promise((resolve, reject) => {
const child = fork(tmpFilePath, { silent: true });
try {
await fs.symlink(
join(
this.getInMemoryLayerFolderPath(serverlessFunction.layerVersion),
'node_modules',
),
join(
inMemoryServerlessFunctionFolderPath,
OUTDIR_FOLDER,
'node_modules',
),
'dir',
);
} catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
}
child.on('message', (message: object | ServerlessExecuteError) => {
const duration = Date.now() - startTime;
async publish(serverlessFunction: ServerlessFunctionEntity) {
const newVersion = serverlessFunction.latestVersion
? `${parseInt(serverlessFunction.latestVersion, 10) + 1}`
: '1';
const draftFolderPath = getServerlessFolder({
serverlessFunction: serverlessFunction,
version: 'draft',
});
const newFolderPath = getServerlessFolder({
serverlessFunction: serverlessFunction,
version: newVersion,
});
await this.fileStorageService.copy({
from: { folderPath: draftFolderPath },
to: { folderPath: newFolderPath },
});
await this.build(serverlessFunction, newVersion);
return newVersion;
}
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object,
version: string,
): Promise<ServerlessExecuteResult> {
const startTime = Date.now();
const computedVersion =
version === 'latest' ? serverlessFunction.latestVersion : version;
const listenerFile = join(
this.getInMemoryServerlessFunctionFolderPath(
serverlessFunction,
computedVersion,
),
OUTDIR_FOLDER,
LISTENER_FILE_NAME,
);
try {
return await new Promise((resolve, reject) => {
const child = fork(listenerFile, { silent: true });
child.on('message', (message: object | ServerlessExecuteError) => {
const duration = Date.now() - startTime;
if ('errorType' in message) {
resolve({
data: null,
duration,
error: message,
status: ServerlessFunctionExecutionStatus.ERROR,
});
} else {
resolve({
data: message,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
});
}
child.kill();
});
child.stderr?.on('data', (data) => {
const stackTrace = data
.toString()
.split('\n')
.filter((line: string) => line.trim() !== '');
const errorTrace = stackTrace.filter((line: string) =>
line.includes('Error: '),
)?.[0];
let errorType = 'Unknown';
let errorMessage = '';
if (errorTrace) {
errorType = errorTrace.split(':')[0];
errorMessage = errorTrace.split(': ')[1];
}
const duration = Date.now() - startTime;
if ('errorType' in message) {
resolve({
data: null,
duration,
error: message,
status: ServerlessFunctionExecutionStatus.ERROR,
error: {
errorType,
errorMessage,
stackTrace: stackTrace,
},
});
} else {
resolve({
data: message,
duration,
status: ServerlessFunctionExecutionStatus.SUCCESS,
});
}
child.kill();
fs.unlink(tmpFilePath).catch(console.error);
});
child.stderr?.on('data', (data) => {
const stackTrace = data
.toString()
.split('\n')
.filter((line: string) => line.trim() !== '');
const errorTrace = stackTrace.filter((line: string) =>
line.includes('Error: '),
)?.[0];
let errorType = 'Unknown';
let errorMessage = '';
if (errorTrace) {
errorType = errorTrace.split(':')[0];
errorMessage = errorTrace.split(': ')[1];
}
const duration = Date.now() - startTime;
resolve({
data: null,
duration,
status: ServerlessFunctionExecutionStatus.ERROR,
error: {
errorType,
errorMessage,
stackTrace: stackTrace,
},
child.kill();
});
child.kill();
fs.unlink(tmpFilePath).catch(console.error);
});
child.on('error', (error) => {
reject(error);
child.kill();
fs.unlink(tmpFilePath).catch(console.error);
});
child.on('error', (error) => {
reject(error);
child.kill();
});
child.on('exit', (code) => {
if (code && code !== 0) {
reject(new Error(`Child process exited with code ${code}`));
fs.unlink(tmpFilePath).catch(console.error);
}
});
child.on('exit', (code) => {
if (code && code !== 0) {
reject(new Error(`Child process exited with code ${code}`));
}
});
child.send({ event: payload });
});
child.send({ event: payload });
});
} catch (error) {
return {
data: null,
duration: Date.now() - startTime,
error: {
errorType: 'UnhandledError',
errorMessage: error.message || 'Unknown error',
stackTrace: error.stack ? error.stack.split('\n') : [],
},
status: ServerlessFunctionExecutionStatus.ERROR,
};
}
}
}

View File

@ -1,6 +1,11 @@
import ts from 'typescript';
import { join } from 'path';
export const compileTypescript = (typescriptCode: string): string => {
import ts, { createProgram } from 'typescript';
import { OUTDIR_FOLDER } from 'src/engine/core-modules/serverless/drivers/constants/outdir-folder';
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
export const compileTypescript = (folderPath: string) => {
const options: ts.CompilerOptions = {
module: ts.ModuleKind.CommonJS,
target: ts.ScriptTarget.ES2017,
@ -8,12 +13,9 @@ export const compileTypescript = (typescriptCode: string): string => {
esModuleInterop: true,
resolveJsonModule: true,
allowSyntheticDefaultImports: true,
outDir: join(folderPath, OUTDIR_FOLDER, 'src'),
types: ['node'],
};
const result = ts.transpileModule(typescriptCode, {
compilerOptions: options,
});
return result.outputText;
createProgram([join(folderPath, 'src', INDEX_FILE_NAME)], options).emit();
};

View File

@ -0,0 +1,39 @@
import path, { join } from 'path';
import fs from 'fs/promises';
import { ASSET_PATH } from 'src/constants/assets-path';
type File = { name: string; path: string; content: Buffer };
const getAllFiles = async (
rootDir: string,
dir: string = rootDir,
files: File[] = [],
): Promise<File[]> => {
const dirEntries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of dirEntries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
return getAllFiles(rootDir, fullPath, files);
} else {
files.push({
path: path.relative(rootDir, dir),
name: entry.name,
content: await fs.readFile(fullPath),
});
}
}
return files;
};
export const getBaseTypescriptProjectFiles = (async () => {
const baseTypescriptProjectPath = join(
ASSET_PATH,
`engine/core-modules/serverless/drivers/constants/base-typescript-project`,
);
return await getAllFiles(baseTypescriptProjectPath);
})();

View File

@ -1,12 +1,17 @@
import path from 'path';
import path, { join } from 'path';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
import { ASSET_PATH } from 'src/constants/assets-path';
// Can only be used in src/engine/integrations/serverless/drivers/utils folder
export const getLayerDependenciesDirName = (
version: 'latest' | 'engine' | number,
): string => {
const formattedVersion = version === 'latest' ? LAST_LAYER_VERSION : version;
return path.resolve(__dirname, `../layers/${formattedVersion}`);
const baseTypescriptProjectPath = join(
ASSET_PATH,
`engine/core-modules/serverless/drivers/layers/${formattedVersion}`,
);
return path.resolve(baseTypescriptProjectPath);
};

View File

@ -10,14 +10,12 @@ export const NODE_LAYER_SUBFOLDER = 'nodejs';
const TEMPORARY_LAMBDA_FOLDER = 'lambda-build';
const TEMPORARY_LAMBDA_SOURCE_FOLDER = 'src';
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
const LAMBDA_ENTRY_FILE_NAME = 'index.js';
export class LambdaBuildDirectoryManager {
private temporaryDir = join(
SERVERLESS_TMPDIR_FOLDER,
`${TEMPORARY_LAMBDA_FOLDER}-${v4()}`,
);
private lambdaHandler = `${LAMBDA_ENTRY_FILE_NAME.split('.')[0]}.handler`;
async init() {
const sourceTemporaryDir = join(
@ -25,15 +23,12 @@ export class LambdaBuildDirectoryManager {
TEMPORARY_LAMBDA_SOURCE_FOLDER,
);
const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
const javascriptFilePath = join(sourceTemporaryDir, LAMBDA_ENTRY_FILE_NAME);
await fs.mkdir(sourceTemporaryDir, { recursive: true });
return {
sourceTemporaryDir,
lambdaZipPath,
javascriptFilePath,
lambdaHandler: this.lambdaHandler,
};
}