admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

6653 serverless functions store and use environment variables in serverless function scripts (#7390)

Browse Source 
![image](https://github.com/user-attachments/assets/a15bd4c1-3db4-4466-b748-06bdf3874354)

![image](https://github.com/user-attachments/assets/71242dfb-956b-43ed-9704-87cb0dfbc98d)
This commit is contained in:
martmull
2024-10-03 13:56:17 +02:00
committed by GitHub
parent 3cd24d542b
commit 62fe1d0e88
39 changed files with 815 additions and 513 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/create-serverless-function-from-file.input.ts
View File

@ -1,16 +0,0 @@
import { Field, InputType } from '@nestjs/graphql';
import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
@InputType()
export class CreateServerlessFunctionFromFileInput {
@IsString()
@IsNotEmpty()
@Field()
name: string;
@IsString()
@IsOptional()
@Field({ nullable: true })
description?: string;
}

13
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input.ts
View File

@ -1,13 +1,16 @@
import { Field, InputType } from '@nestjs/graphql';
import { IsNotEmpty, IsString } from 'class-validator';
import { CreateServerlessFunctionFromFileInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function-from-file.input';
import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
@InputType()
export class CreateServerlessFunctionInput extends CreateServerlessFunctionFromFileInput {
export class CreateServerlessFunctionInput {
@IsString()
@IsNotEmpty()
@Field()
code: string;
name: string;
@IsString()
@IsOptional()
@Field({ nullable: true })
description?: string;
}

5
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/serverless-function.dto.ts
View File

@ -50,11 +50,6 @@ export class ServerlessFunctionDTO {
@Field({ nullable: true })
description: string;
@IsString()
@IsNotEmpty()
@Field()
sourceCodeHash: string;
@IsString()
@IsNotEmpty()
@Field()

9
packages/twenty-server/src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input.ts
View File

@ -1,6 +1,7 @@
import { Field, InputType } from '@nestjs/graphql';
import { IsNotEmpty, IsString, IsUUID } from 'class-validator';
import { IsNotEmpty, IsObject, IsString, IsUUID } from 'class-validator';
import graphqlTypeJson from 'graphql-type-json';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@ -21,7 +22,7 @@ export class UpdateServerlessFunctionInput {
@Field({ nullable: true })
description?: string;
@IsString()
@Field()
code: string;
@Field(() => graphqlTypeJson)
@IsObject()
code: JSON;
}

3
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.entity.ts
View File

@ -29,9 +29,6 @@ export class ServerlessFunctionEntity {
@Column({ nullable: true })
latestVersion: string;
@Column({ nullable: false })
sourceCodeHash: string;
@Column({ nullable: false, default: ServerlessFunctionRuntime.NODE18 })
runtime: ServerlessFunctionRuntime;

26
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.resolver.ts
View File

@ -3,7 +3,6 @@ import { Args, Mutation, Query, Resolver } from '@nestjs/graphql';
import { InjectRepository } from '@nestjs/typeorm';
import graphqlTypeJson from 'graphql-type-json';
import { FileUpload, GraphQLUpload } from 'graphql-upload';
import { Repository } from 'typeorm';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
@ -11,7 +10,6 @@ import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { CreateServerlessFunctionFromFileInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function-from-file.input';
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { DeleteServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/delete-serverless-function.input';
import { ExecuteServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/execute-serverless-function.input';
@ -63,7 +61,7 @@ export class ServerlessFunctionResolver {
}
}
@Query(() => String, { nullable: true })
@Query(() => graphqlTypeJson, { nullable: true })
async getServerlessFunctionSourceCode(
@Args('input') input: GetServerlessFunctionSourceCodeInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
@ -130,28 +128,6 @@ export class ServerlessFunctionResolver {
name: input.name,
description: input.description,
},
input.code,
workspaceId,
);
} catch (error) {
serverlessFunctionGraphQLApiExceptionHandler(error);
}
}
@Mutation(() => ServerlessFunctionDTO)
async createOneServerlessFunctionFromFile(
@Args({ name: 'file', type: () => GraphQLUpload })
file: FileUpload,
@Args('input')
input: CreateServerlessFunctionFromFileInput,
@AuthWorkspace() { id: workspaceId }: Workspace,
) {
try {
await this.checkFeatureFlag(workspaceId);
return await this.serverlessFunctionService.createOneServerlessFunction(
input,
file,
workspaceId,
);
} catch (error) {

91
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
View File

@ -1,9 +1,11 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { basename, dirname, join } from 'path';
import { TypeOrmQueryService } from '@ptc-org/nestjs-query-typeorm';
import { FileUpload } from 'graphql-upload';
import { Repository } from 'typeorm';
import deepEqual from 'deep-equal';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
@ -12,10 +14,9 @@ import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.se
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { SOURCE_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/source-file-name';
import { INDEX_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/index-file-name';
import { ServerlessService } from 'src/engine/core-modules/serverless/serverless.service';
import { getServerlessFolder } from 'src/engine/core-modules/serverless/utils/serverless-get-folder.utils';
import { CreateServerlessFunctionFromFileInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function-from-file.input';
import { UpdateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/update-serverless-function.input';
import {
ServerlessFunctionEntity,
@ -25,10 +26,12 @@ import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
import { isDefined } from 'src/utils/is-defined';
import { getLastLayerDependencies } from 'src/engine/core-modules/serverless/drivers/utils/get-last-layer-dependencies';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/serverless/drivers/layers/last-layer-version';
import { CreateServerlessFunctionInput } from 'src/engine/metadata-modules/serverless-function/dtos/create-serverless-function.input';
import { getBaseTypescriptProjectFiles } from 'src/engine/core-modules/serverless/drivers/utils/get-base-typescript-project-files';
import { ENV_FILE_NAME } from 'src/engine/core-modules/serverless/drivers/constants/env-file-name';
@Injectable()
export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFunctionEntity> {
@ -47,7 +50,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
workspaceId: string,
id: string,
version: string,
) {
): Promise<{ [filePath: string]: string } | undefined> {
const serverlessFunction = await this.serverlessFunctionRepository.findOne({
where: {
id,
@ -68,12 +71,20 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
version,
});
const fileStream = await this.fileStorageService.read({
folderPath,
filename: SOURCE_FILE_NAME,
const indexFileStream = await this.fileStorageService.read({
folderPath: join(folderPath, 'src'),
filename: INDEX_FILE_NAME,
});
return await readFileContent(fileStream);
const envFileStream = await this.fileStorageService.read({
folderPath: folderPath,
filename: ENV_FILE_NAME,
});
return {
'.env': await readFileContent(envFileStream),
'src/index.ts': await readFileContent(indexFileStream),
};
} catch (error) {
if (error.code === FileStorageExceptionCode.FILE_NOT_FOUND) {
return;
@ -132,10 +143,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
'draft',
);
if (
serverlessFunctionCreateHash(latestCode || '') ===
serverlessFunctionCreateHash(draftCode || '')
) {
if (deepEqual(latestCode, draftCode)) {
throw new Error(
'Cannot publish a new version when code has not changed',
);
@ -146,20 +154,6 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
existingServerlessFunction,
);
const draftFolderPath = getServerlessFolder({
serverlessFunction: existingServerlessFunction,
version: 'draft',
});
const newFolderPath = getServerlessFolder({
serverlessFunction: existingServerlessFunction,
version: newVersion,
});
await this.fileStorageService.copy({
from: { folderPath: draftFolderPath },
to: { folderPath: newFolderPath },
});
await super.updateOne(existingServerlessFunction.id, {
latestVersion: newVersion,
});
@ -213,9 +207,6 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
name: serverlessFunctionInput.name,
description: serverlessFunctionInput.description,
syncStatus: ServerlessFunctionSyncStatus.NOT_READY,
sourceCodeHash: serverlessFunctionCreateHash(
serverlessFunctionInput.code,
),
});
const fileFolder = getServerlessFolder({
@ -223,12 +214,14 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
version: 'draft',
});
await this.fileStorageService.write({
file: serverlessFunctionInput.code,
name: SOURCE_FILE_NAME,
mimeType: undefined,
folder: fileFolder,
});
for (const key of Object.keys(serverlessFunctionInput.code)) {
await this.fileStorageService.write({
file: serverlessFunctionInput.code[key],
name: basename(key),
mimeType: undefined,
folder: join(fileFolder, dirname(key)),
});
}
await this.serverlessService.build(existingServerlessFunction, 'draft');
await super.updateOne(existingServerlessFunction.id, {
@ -259,22 +252,12 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
}
async createOneServerlessFunction(
serverlessFunctionInput: CreateServerlessFunctionFromFileInput,
code: FileUpload | string,
serverlessFunctionInput: CreateServerlessFunctionInput,
workspaceId: string,
) {
let typescriptCode: string;
if (typeof code === 'string') {
typescriptCode = code;
} else {
typescriptCode = await readFileContent(code.createReadStream());
}
const createdServerlessFunction = await super.createOne({
...serverlessFunctionInput,
workspaceId,
sourceCodeHash: serverlessFunctionCreateHash(typescriptCode),
layerVersion: LAST_LAYER_VERSION,
});
@ -283,12 +266,14 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
version: 'draft',
});
await this.fileStorageService.write({
file: typescriptCode,
name: SOURCE_FILE_NAME,
mimeType: undefined,
folder: draftFileFolder,
});
for (const file of await getBaseTypescriptProjectFiles) {
await this.fileStorageService.write({
file: file.content,
name: file.name,
mimeType: undefined,
folder: join(draftFileFolder, file.path),
});
}
await this.serverlessService.build(createdServerlessFunction, 'draft');