Serverless function improvements (#6769)

- add layer for lambda execution
- add layer for local execution
- add package resolve for the monaco editor
- add route to get installed package for serverless functions
- add layer versioning
This commit is contained in:
martmull
2024-09-02 15:25:20 +02:00
committed by GitHub
parent f8890689ee
commit 7e03419c16
41 changed files with 4834 additions and 164 deletions

View File

@ -39,6 +39,7 @@ const documents = {
"\n mutation ExecuteOneServerlessFunction(\n $input: ExecuteServerlessFunctionInput!\n ) {\n executeOneServerlessFunction(input: $input) {\n data\n duration\n status\n error\n }\n }\n": types.ExecuteOneServerlessFunctionDocument,
"\n \n mutation PublishOneServerlessFunction(\n $input: PublishServerlessFunctionInput!\n ) {\n publishServerlessFunction(input: $input) {\n ...ServerlessFunctionFields\n }\n }\n": types.PublishOneServerlessFunctionDocument,
"\n \n mutation UpdateOneServerlessFunction($input: UpdateServerlessFunctionInput!) {\n updateOneServerlessFunction(input: $input) {\n ...ServerlessFunctionFields\n }\n }\n": types.UpdateOneServerlessFunctionDocument,
"\n query FindManyAvailablePackages {\n getAvailablePackages\n }\n": types.FindManyAvailablePackagesDocument,
"\n \n query GetManyServerlessFunctions {\n serverlessFunctions(paging: { first: 100 }) {\n edges {\n node {\n ...ServerlessFunctionFields\n }\n }\n }\n }\n": types.GetManyServerlessFunctionsDocument,
"\n \n query GetOneServerlessFunction($id: UUID!) {\n serverlessFunction(id: $id) {\n ...ServerlessFunctionFields\n }\n }\n": types.GetOneServerlessFunctionDocument,
"\n query FindOneServerlessFunctionSourceCode(\n $input: GetServerlessFunctionSourceCodeInput!\n ) {\n getServerlessFunctionSourceCode(input: $input)\n }\n": types.FindOneServerlessFunctionSourceCodeDocument,
@ -162,6 +163,10 @@ export function graphql(source: "\n \n mutation PublishOneServerlessFunction(\
* The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/
export function graphql(source: "\n \n mutation UpdateOneServerlessFunction($input: UpdateServerlessFunctionInput!) {\n updateOneServerlessFunction(input: $input) {\n ...ServerlessFunctionFields\n }\n }\n"): (typeof documents)["\n \n mutation UpdateOneServerlessFunction($input: UpdateServerlessFunctionInput!) {\n updateOneServerlessFunction(input: $input) {\n ...ServerlessFunctionFields\n }\n }\n"];
/**
* The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/
export function graphql(source: "\n query FindManyAvailablePackages {\n getAvailablePackages\n }\n"): (typeof documents)["\n query FindManyAvailablePackages {\n getAvailablePackages\n }\n"];
/**
* The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients.
*/

View File

@ -314,7 +314,7 @@ export type ExecuteServerlessFunctionInput = {
/** Id of the serverless function to execute */
id: Scalars['UUID']['input'];
/** Payload in JSON format */
payload?: InputMaybe<Scalars['JSON']['input']>;
payload: Scalars['JSON']['input'];
/** Version of the serverless function to execute */
version?: Scalars['String']['input'];
};
@ -814,9 +814,10 @@ export type Query = {
findOneRemoteServerById: RemoteServer;
findWorkspaceFromInviteHash: Workspace;
getAISQLQuery: AisqlQueryResult;
getAvailablePackages: Scalars['JSON']['output'];
getPostgresCredentials?: Maybe<PostgresCredentials>;
getProductPrices: ProductPricesEntity;
getServerlessFunctionSourceCode: Scalars['String']['output'];
getServerlessFunctionSourceCode?: Maybe<Scalars['String']['output']>;
getTimelineCalendarEventsFromCompanyId: TimelineCalendarEventsWithTotal;
getTimelineCalendarEventsFromPersonId: TimelineCalendarEventsWithTotal;
getTimelineThreadsFromCompanyId: TimelineThreadsWithTotal;
@ -1761,6 +1762,11 @@ export type UpdateOneServerlessFunctionMutationVariables = Exact<{
export type UpdateOneServerlessFunctionMutation = { __typename?: 'Mutation', updateOneServerlessFunction: { __typename?: 'ServerlessFunction', id: any, name: string, description?: string | null, sourceCodeHash: string, runtime: string, syncStatus: ServerlessFunctionSyncStatus, latestVersion?: string | null, createdAt: any, updatedAt: any } };
export type FindManyAvailablePackagesQueryVariables = Exact<{ [key: string]: never; }>;
export type FindManyAvailablePackagesQuery = { __typename?: 'Query', getAvailablePackages: any };
export type GetManyServerlessFunctionsQueryVariables = Exact<{ [key: string]: never; }>;
@ -1778,7 +1784,7 @@ export type FindOneServerlessFunctionSourceCodeQueryVariables = Exact<{
}>;
export type FindOneServerlessFunctionSourceCodeQuery = { __typename?: 'Query', getServerlessFunctionSourceCode: string };
export type FindOneServerlessFunctionSourceCodeQuery = { __typename?: 'Query', getServerlessFunctionSourceCode?: string | null };
export const RemoteServerFieldsFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"RemoteServerFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"RemoteServer"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"foreignDataWrapperId"}},{"kind":"Field","name":{"kind":"Name","value":"foreignDataWrapperOptions"}},{"kind":"Field","name":{"kind":"Name","value":"foreignDataWrapperType"}},{"kind":"Field","name":{"kind":"Name","value":"userMappingOptions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"user"}}]}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"schema"}},{"kind":"Field","name":{"kind":"Name","value":"label"}}]}}]} as unknown as DocumentNode<RemoteServerFieldsFragment, unknown>;
export const RemoteTableFieldsFragmentDoc = {"kind":"Document","definitions":[{"kind":"FragmentDefinition","name":{"kind":"Name","value":"RemoteTableFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"RemoteTable"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"schema"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"schemaPendingUpdates"}}]}}]} as unknown as DocumentNode<RemoteTableFieldsFragment, unknown>;
@ -1806,6 +1812,7 @@ export const DeleteOneServerlessFunctionDocument = {"kind":"Document","definitio
export const ExecuteOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ExecuteOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ExecuteServerlessFunctionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"executeOneServerlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"data"}},{"kind":"Field","name":{"kind":"Name","value":"duration"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"error"}}]}}]}}]} as unknown as DocumentNode<ExecuteOneServerlessFunctionMutation, ExecuteOneServerlessFunctionMutationVariables>;
export const PublishOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"PublishOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"PublishServerlessFunctionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"publishServerlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<PublishOneServerlessFunctionMutation, PublishOneServerlessFunctionMutationVariables>;
export const UpdateOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UpdateServerlessFunctionInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateOneServerlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<UpdateOneServerlessFunctionMutation, UpdateOneServerlessFunctionMutationVariables>;
export const FindManyAvailablePackagesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindManyAvailablePackages"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getAvailablePackages"}}]}}]} as unknown as DocumentNode<FindManyAvailablePackagesQuery, FindManyAvailablePackagesQueryVariables>;
export const GetManyServerlessFunctionsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetManyServerlessFunctions"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverlessFunctions"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"paging"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"first"},"value":{"kind":"IntValue","value":"100"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"edges"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"node"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<GetManyServerlessFunctionsQuery, GetManyServerlessFunctionsQueryVariables>;
export const GetOneServerlessFunctionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetOneServerlessFunction"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UUID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"serverlessFunction"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"ServerlessFunctionFields"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"ServerlessFunctionFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"ServerlessFunction"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"sourceCodeHash"}},{"kind":"Field","name":{"kind":"Name","value":"runtime"}},{"kind":"Field","name":{"kind":"Name","value":"syncStatus"}},{"kind":"Field","name":{"kind":"Name","value":"latestVersion"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}}]}}]} as unknown as DocumentNode<GetOneServerlessFunctionQuery, GetOneServerlessFunctionQueryVariables>;
export const FindOneServerlessFunctionSourceCodeDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"FindOneServerlessFunctionSourceCode"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"GetServerlessFunctionSourceCodeInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"getServerlessFunctionSourceCode"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}]}]}}]} as unknown as DocumentNode<FindOneServerlessFunctionSourceCodeQuery, FindOneServerlessFunctionSourceCodeQueryVariables>;

View File

@ -0,0 +1,32 @@
import { useEffect, useState } from 'react';
import { useDebouncedCallback } from 'use-debounce';
export const usePreventOverlapCallback = (
callback: () => Promise<void>,
wait?: number,
) => {
const [isRunning, setIsRunning] = useState(false);
const [pendingRun, setPendingRun] = useState(false);
const handleCallback = async () => {
if (isRunning) {
setPendingRun(true);
return;
}
setIsRunning(true);
try {
await callback();
} finally {
setIsRunning(false);
}
};
useEffect(() => {
if (!isRunning && pendingRun) {
setPendingRun(false);
callback();
}
}, [callback, isRunning, pendingRun, setPendingRun]);
return useDebouncedCallback(handleCallback, wait);
};

View File

@ -108,6 +108,7 @@ export const SettingsNavigationDrawerItems = () => {
/>
{accountSubSettings.map((navigationItem, index) => (
<SettingsNavigationDrawerItem
key={index}
label={navigationItem.label}
path={navigationItem.path}
Icon={navigationItem.Icon}

View File

@ -0,0 +1,7 @@
import { gql } from '@apollo/client';
export const FIND_MANY_AVAILABLE_PACKAGES = gql`
query FindManyAvailablePackages {
getAvailablePackages
}
`;

View File

@ -0,0 +1,20 @@
import { useApolloMetadataClient } from '@/object-metadata/hooks/useApolloMetadataClient';
import { useQuery } from '@apollo/client';
import { FIND_MANY_AVAILABLE_PACKAGES } from '@/settings/serverless-functions/graphql/queries/findManyAvailablePackages';
import {
FindManyAvailablePackagesQuery,
FindManyAvailablePackagesQueryVariables,
} from '~/generated-metadata/graphql';
export const useGetAvailablePackages = () => {
const apolloMetadataClient = useApolloMetadataClient();
const { data } = useQuery<
FindManyAvailablePackagesQuery,
FindManyAvailablePackagesQueryVariables
>(FIND_MANY_AVAILABLE_PACKAGES, {
client: apolloMetadataClient ?? undefined,
});
return {
availablePackages: data?.getAvailablePackages || null,
};
};

View File

@ -1,9 +1,12 @@
import Editor, { Monaco, EditorProps } from '@monaco-editor/react';
import { AutoTypings } from 'monaco-editor-auto-typings';
import { editor, MarkerSeverity } from 'monaco-editor';
import { codeEditorTheme } from '@/ui/input/code-editor/theme/CodeEditorTheme';
import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { useEffect } from 'react';
import { useGetAvailablePackages } from '@/settings/serverless-functions/hooks/useGetAvailablePackages';
import { isDefined } from '~/utils/isDefined';
export const DEFAULT_CODE = `export const handler = async (
event: object,
@ -38,12 +41,24 @@ export const CodeEditor = ({
}: CodeEditorProps) => {
const theme = useTheme();
const handleEditorDidMount = (
const { availablePackages } = useGetAvailablePackages();
const handleEditorDidMount = async (
editor: editor.IStandaloneCodeEditor,
monaco: Monaco,
) => {
monaco.editor.defineTheme('codeEditorTheme', codeEditorTheme(theme));
monaco.editor.setTheme('codeEditorTheme');
if (language === 'typescript') {
await AutoTypings.create(editor, {
monaco,
preloadPackages: true,
onlySpecifiedPackages: true,
versions: availablePackages,
debounceDuration: 0,
});
}
};
const handleEditorValidation = (markers: editor.IMarker[]) => {
@ -68,28 +83,31 @@ export const CodeEditor = ({
document.head.removeChild(style);
};
}, []);
return (
<div>
{header}
<StyledEditor
height={height}
language={language}
value={value}
onMount={handleEditorDidMount}
onChange={(value?: string) => value && onChange?.(value)}
onValidate={handleEditorValidation}
options={{
...options,
overviewRulerLanes: 0,
scrollbar: {
vertical: 'hidden',
horizontal: 'hidden',
},
minimap: {
enabled: false,
},
}}
/>
</div>
isDefined(availablePackages) && (
<>
{header}
<StyledEditor
height={height}
language={language}
value={value}
onMount={handleEditorDidMount}
onChange={(value?: string) => value && onChange?.(value)}
onValidate={handleEditorValidation}
options={{
...options,
overviewRulerLanes: 0,
scrollbar: {
vertical: 'hidden',
horizontal: 'hidden',
},
minimap: {
enabled: false,
},
}}
/>
</>
)
);
};

View File

@ -20,10 +20,9 @@ import { useParams } from 'react-router-dom';
import { useRecoilValue, useSetRecoilState } from 'recoil';
import { IconCode, IconFunction, IconSettings, IconTestPipe } from 'twenty-ui';
import { isDefined } from '~/utils/isDefined';
import { useDebouncedCallback } from 'use-debounce';
import { useGetOneServerlessFunctionSourceCode } from '@/settings/serverless-functions/hooks/useGetOneServerlessFunctionSourceCode';
import { useState } from 'react';
import isEmpty from 'lodash.isempty';
import { usePreventOverlapCallback } from '~/hooks/usePreventOverlapCallback';
const TAB_LIST_COMPONENT_ID = 'serverless-function-detail';
@ -53,9 +52,6 @@ export const SettingsServerlessFunctionDetail = () => {
const save = async () => {
try {
if (isEmpty(formValues.name)) {
return;
}
await updateOneServerlessFunction({
id: serverlessFunctionId,
name: formValues.name,
@ -72,7 +68,7 @@ export const SettingsServerlessFunctionDetail = () => {
}
};
const handleSave = useDebouncedCallback(save, 500);
const handleSave = usePreventOverlapCallback(save, 1000);
const onChange = (key: string) => {
return async (value: string | undefined) => {

View File

@ -127,5 +127,11 @@ export default defineConfig(({ command, mode }) => {
localsConvention: 'camelCaseOnly',
},
},
resolve: {
alias: {
path: 'rollup-plugin-node-polyfills/polyfills/path',
},
},
};
});

View File

@ -4,6 +4,21 @@
"sourceRoot": "src",
"compilerOptions": {
"builder": "swc",
"typeCheck": true
"typeCheck": true,
"assets": [
{
"include": "**/serverless/drivers/layers/*/package.json",
"outDir": "dist/src"
},
{
"include": "**/serverless/drivers/layers/*/yarn.lock",
"outDir": "dist/src"
},
{
"include": "**/serverless/drivers/layers/engine/**",
"outDir": "dist/src"
}
],
"watchAssets": true
}
}

View File

@ -15,6 +15,7 @@
"typeorm": "../../node_modules/typeorm/.bin/typeorm"
},
"dependencies": {
"@esbuild-plugins/node-modules-polyfill": "^0.2.2",
"@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch",
"@langchain/mistralai": "^0.0.24",
"@langchain/openai": "^0.1.3",
@ -37,11 +38,13 @@
"lodash.omitby": "^4.6.0",
"lodash.uniq": "^4.5.0",
"lodash.uniqby": "^4.7.0",
"monaco-editor": "^0.50.0",
"monaco-editor": "^0.51.0",
"monaco-editor-auto-typings": "^0.4.5",
"passport": "^0.7.0",
"psl": "^1.9.0",
"tsconfig-paths": "^4.2.0",
"typeorm": "patch:typeorm@0.3.20#./patches/typeorm+0.3.20.patch",
"unzipper": "^0.12.3",
"zod-to-json-schema": "^3.23.1"
},
"devDependencies": {
@ -59,6 +62,7 @@
"@types/lodash.uniqby": "^4.7.9",
"@types/lodash.upperfirst": "^4.3.7",
"@types/react": "^18.2.39",
"@types/unzipper": "^0",
"rimraf": "^5.0.5",
"typescript": "5.3.3"
},

View File

@ -0,0 +1,19 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddServerlessFunctionLayerVersionColumn1724946099627
implements MigrationInterface
{
name = 'AddServerlessFunctionLayerVersionColumn1724946099627';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."serverlessFunction" ADD "layerVersion" integer`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "metadata"."serverlessFunction" DROP COLUMN "layerVersion"`,
);
}
}

View File

@ -127,6 +127,9 @@ export class LocalDriver implements StorageDriver {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
if (!params.from.filename && params.to.filename) {
throw new Error('Cannot copy folder to file');
}
const fromPath = join(
`${this.options.storagePath}/`,
params.from.folderPath,
@ -139,6 +142,8 @@ export class LocalDriver implements StorageDriver {
params.to.filename || '',
);
await this.createFolder(dirname(toPath));
try {
await fs.cp(fromPath, toPath, { recursive: true });
} catch (error) {

View File

@ -20,6 +20,8 @@ import {
FileStorageExceptionCode,
} from 'src/engine/integrations/file-storage/interfaces/file-storage-exception';
import { isDefined } from 'src/utils/is-defined';
import { StorageDriver } from './interfaces/storage-driver.interface';
export interface S3DriverOptions extends S3ClientConfig {
@ -191,35 +193,80 @@ export class S3Driver implements StorageDriver {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };
}): Promise<void> {
if (!params.from.filename && params.to.filename) {
throw new Error('Cannot copy folder to file');
}
const fromKey = `${params.from.folderPath}/${params.from.filename || ''}`;
const toKey = `${params.to.folderPath}/${params.to.filename || ''}`;
try {
// Check if the source file exists
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: fromKey,
}),
if (isDefined(params.from.filename)) {
try {
// Check if the source file exists
await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: fromKey,
}),
);
// Copy the object to the new location
await this.s3Client.send(
new CopyObjectCommand({
CopySource: `${this.bucketName}/${fromKey}`,
Bucket: this.bucketName,
Key: toKey,
}),
);
return;
} catch (error) {
if (error.name === 'NotFound') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
}
// For other errors, throw the original error
throw error;
}
}
const listedObjects = await this.s3Client.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: fromKey,
}),
);
if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
throw new Error('No objects found in the source folder.');
}
for (const object of listedObjects.Contents) {
const match = object.Key?.match(/(.*)\/(.*)/);
if (!isDefined(match)) {
continue;
}
const fromFolderPath = match[1];
const filename = match[2];
const toFolderPath = fromFolderPath.replace(
params.from.folderPath,
params.to.folderPath,
);
// Copy the object to the new location
await this.s3Client.send(
new CopyObjectCommand({
CopySource: `${this.bucketName}/${fromKey}`,
Bucket: this.bucketName,
Key: toKey,
}),
);
} catch (error) {
if (error.name === 'NotFound') {
throw new FileStorageException(
'File not found',
FileStorageExceptionCode.FILE_NOT_FOUND,
);
if (!isDefined(toFolderPath)) {
continue;
}
// For other errors, throw the original error
throw error;
await this.copy({
from: {
folderPath: fromFolderPath,
filename,
},
to: { folderPath: toFolderPath, filename },
});
}
}

View File

@ -17,7 +17,6 @@ import { LLMTracingModule } from 'src/engine/integrations/llm-tracing/llm-tracin
import { llmTracingModuleFactory } from 'src/engine/integrations/llm-tracing/llm-tracing.module-factory';
import { loggerModuleFactory } from 'src/engine/integrations/logger/logger.module-factory';
import { messageQueueModuleFactory } from 'src/engine/integrations/message-queue/message-queue.module-factory';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
import { serverlessModuleFactory } from 'src/engine/integrations/serverless/serverless-module.factory';
import { ServerlessModule } from 'src/engine/integrations/serverless/serverless.module';
@ -68,11 +67,7 @@ import { MessageQueueModule } from './message-queue/message-queue.module';
}),
ServerlessModule.forRootAsync({
useFactory: serverlessModuleFactory,
inject: [
EnvironmentService,
FileStorageService,
BuildDirectoryManagerService,
],
inject: [EnvironmentService, FileStorageService],
}),
],
exports: [],

View File

@ -0,0 +1 @@
export const COMMON_LAYER_NAME = 'common-layer';

View File

@ -0,0 +1,4 @@
import { join } from 'path';
import { tmpdir } from 'os';
export const SERVERLESS_TMPDIR_FOLDER = join(tmpdir(), 'serverless-tmpdir');

View File

@ -23,7 +23,7 @@ export interface ServerlessDriver {
publish(serverlessFunction: ServerlessFunctionEntity): Promise<string>;
execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined,
payload: object,
version: string,
): Promise<ServerlessExecuteResult>;
}

View File

@ -1,17 +1,23 @@
import fs from 'fs';
import * as fs from 'fs/promises';
import { join } from 'path';
import {
CreateFunctionCommand,
DeleteFunctionCommand,
GetFunctionCommand,
InvokeCommand,
InvokeCommandInput,
Lambda,
LambdaClientConfig,
PublishLayerVersionCommand,
PublishLayerVersionCommandInput,
PublishVersionCommand,
PublishVersionCommandInput,
ResourceNotFoundException,
UpdateFunctionCodeCommand,
waitUntilFunctionUpdatedV2,
ListLayerVersionsCommandInput,
ListLayerVersionsCommand,
} from '@aws-sdk/client-lambda';
import { CreateFunctionCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/CreateFunctionCommand';
import { UpdateFunctionCodeCommandInput } from '@aws-sdk/client-lambda/dist-types/commands/UpdateFunctionCodeCommand';
@ -21,20 +27,28 @@ import {
ServerlessExecuteResult,
} from 'src/engine/integrations/serverless/drivers/interfaces/serverless-driver.interface';
import {
ServerlessFunctionEntity,
ServerlessFunctionRuntime,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import {
LambdaBuildDirectoryManager,
NODE_LAYER_SUBFOLDER,
} from 'src/engine/integrations/serverless/drivers/utils/lambda-build-directory-manager';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { BaseServerlessDriver } from 'src/engine/integrations/serverless/drivers/base-serverless.driver';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
import { createZipFile } from 'src/engine/integrations/serverless/drivers/utils/create-zip-file';
import { ServerlessFunctionExecutionStatus } from 'src/engine/metadata-modules/serverless-function/dtos/serverless-function-execution-result.dto';
import { ServerlessFunctionEntity } from 'src/engine/metadata-modules/serverless-function/serverless-function.entity';
import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { isDefined } from 'src/utils/is-defined';
import { COMMON_LAYER_NAME } from 'src/engine/integrations/serverless/drivers/constants/common-layer-name';
import { copyAndBuildDependencies } from 'src/engine/integrations/serverless/drivers/utils/copy-and-build-dependencies';
export interface LambdaDriverOptions extends LambdaClientConfig {
fileStorageService: FileStorageService;
buildDirectoryManagerService: BuildDirectoryManagerService;
region: string;
role: string;
}
@ -46,7 +60,6 @@ export class LambdaDriver
private readonly lambdaClient: Lambda;
private readonly lambdaRole: string;
private readonly fileStorageService: FileStorageService;
private readonly buildDirectoryManagerService: BuildDirectoryManagerService;
constructor(options: LambdaDriverOptions) {
super();
@ -55,7 +68,69 @@ export class LambdaDriver
this.lambdaClient = new Lambda({ ...lambdaOptions, region });
this.lambdaRole = role;
this.fileStorageService = options.fileStorageService;
this.buildDirectoryManagerService = options.buildDirectoryManagerService;
}
private async waitFunctionUpdates(
serverlessFunctionId: string,
maxWaitTime: number,
) {
const waitParams = { FunctionName: serverlessFunctionId };
await waitUntilFunctionUpdatedV2(
{ client: this.lambdaClient, maxWaitTime },
waitParams,
);
}
private async createLayerIfNotExists(version: number): Promise<string> {
const listLayerParams: ListLayerVersionsCommandInput = {
LayerName: COMMON_LAYER_NAME,
MaxItems: 1,
};
const listLayerCommand = new ListLayerVersionsCommand(listLayerParams);
const listLayerResult = await this.lambdaClient.send(listLayerCommand);
if (
isDefined(listLayerResult.LayerVersions) &&
listLayerResult.LayerVersions?.[0].Description === `${version}` &&
isDefined(listLayerResult.LayerVersions[0].LayerVersionArn)
) {
return listLayerResult.LayerVersions[0].LayerVersionArn;
}
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const { sourceTemporaryDir, lambdaZipPath } =
await lambdaBuildDirectoryManager.init();
const nodeDependenciesFolder = join(
sourceTemporaryDir,
NODE_LAYER_SUBFOLDER,
);
await copyAndBuildDependencies(nodeDependenciesFolder);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const params: PublishLayerVersionCommandInput = {
LayerName: COMMON_LAYER_NAME,
Content: {
ZipFile: await fs.readFile(lambdaZipPath),
},
CompatibleRuntimes: [ServerlessFunctionRuntime.NODE18],
Description: `${version}`,
};
const command = new PublishLayerVersionCommand(params);
const result = await this.lambdaClient.send(command);
await lambdaBuildDirectoryManager.clean();
if (!isDefined(result.LayerVersionArn)) {
throw new Error('new layer version arn si undefined');
}
return result.LayerVersionArn;
}
private async checkFunctionExists(functionName: string): Promise<boolean> {
@ -95,14 +170,16 @@ export class LambdaDriver
this.fileStorageService,
);
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const {
sourceTemporaryDir,
lambdaZipPath,
javascriptFilePath,
lambdaHandler,
} = await this.buildDirectoryManagerService.init();
} = await lambdaBuildDirectoryManager.init();
await fs.promises.writeFile(javascriptFilePath, javascriptCode);
await fs.writeFile(javascriptFilePath, javascriptCode);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
@ -111,12 +188,17 @@ export class LambdaDriver
);
if (!functionExists) {
const layerArn = await this.createLayerIfNotExists(
serverlessFunction.layerVersion,
);
const params: CreateFunctionCommandInput = {
Code: {
ZipFile: await fs.promises.readFile(lambdaZipPath),
ZipFile: await fs.readFile(lambdaZipPath),
},
FunctionName: serverlessFunction.id,
Handler: lambdaHandler,
Layers: [layerArn],
Role: this.lambdaRole,
Runtime: serverlessFunction.runtime,
Description: 'Lambda function to run user script',
@ -128,7 +210,7 @@ export class LambdaDriver
await this.lambdaClient.send(command);
} else {
const params: UpdateFunctionCodeCommandInput = {
ZipFile: await fs.promises.readFile(lambdaZipPath),
ZipFile: await fs.readFile(lambdaZipPath),
FunctionName: serverlessFunction.id,
};
@ -137,14 +219,9 @@ export class LambdaDriver
await this.lambdaClient.send(command);
}
const waitParams = { FunctionName: serverlessFunction.id };
await this.waitFunctionUpdates(serverlessFunction.id, 10);
await waitUntilFunctionUpdatedV2(
{ client: this.lambdaClient, maxWaitTime: 5 },
waitParams,
);
await this.buildDirectoryManagerService.clean();
await lambdaBuildDirectoryManager.clean();
}
async publish(serverlessFunction: ServerlessFunctionEntity) {
@ -167,7 +244,7 @@ export class LambdaDriver
async execute(
functionToExecute: ServerlessFunctionEntity,
payload: object | undefined = undefined,
payload: object,
version: string,
): Promise<ServerlessExecuteResult> {
const computedVersion =
@ -177,8 +254,11 @@ export class LambdaDriver
computedVersion === 'draft'
? functionToExecute.id
: `${functionToExecute.id}:${computedVersion}`;
await this.waitFunctionUpdates(functionToExecute.id, 10);
const startTime = Date.now();
const params = {
const params: InvokeCommandInput = {
FunctionName: functionName,
Payload: JSON.stringify(payload),
};

View File

@ -0,0 +1,48 @@
{
"dependencies": {
"@types/bcrypt": "^5.0.2",
"@types/deep-equal": "^1.0.4",
"@types/lodash.camelcase": "^4.3.9",
"@types/lodash.compact": "^3.0.9",
"@types/lodash.debounce": "^4.0.9",
"@types/lodash.groupby": "^4.6.9",
"@types/lodash.identity": "^3.0.9",
"@types/lodash.isempty": "^4.4.9",
"@types/lodash.isequal": "^4.5.8",
"@types/lodash.isobject": "^3.0.9",
"@types/lodash.kebabcase": "^4.1.9",
"@types/lodash.mapvalues": "^4.6.9",
"@types/lodash.omit": "^4.5.9",
"@types/lodash.pickby": "^4.6.9",
"@types/lodash.snakecase": "^4.1.9",
"@types/lodash.upperfirst": "^4.3.9",
"@types/uuid": "^10.0.0",
"archiver": "^7.0.1",
"axios": "^1.7.5",
"bcrypt": "^5.1.1",
"body-parser": "^1.20.2",
"deep-equal": "^2.2.3",
"jsonwebtoken": "^9.0.2",
"lodash.camelcase": "^4.3.0",
"lodash.chunk": "^4.2.0",
"lodash.compact": "^3.0.1",
"lodash.debounce": "^4.0.8",
"lodash.groupby": "^4.6.0",
"lodash.identity": "^3.0.0",
"lodash.isempty": "^4.4.0",
"lodash.isequal": "^4.5.0",
"lodash.isobject": "^3.0.2",
"lodash.kebabcase": "^4.1.1",
"lodash.mapvalues": "^4.6.0",
"lodash.merge": "^4.6.2",
"lodash.omit": "^4.5.0",
"lodash.pick": "^4.4.0",
"lodash.pickby": "^4.6.0",
"lodash.snakecase": "^4.1.1",
"lodash.upperfirst": "^4.3.1",
"nodemailer": "^6.9.14",
"sharp": "^0.33.5",
"uuid": "^10.0.0",
"winston": "^3.14.2"
}
}

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,5 @@
enableInlineHunks: true
nodeLinker: node-modules
yarnPath: .yarn/releases/yarn-4.4.0.cjs

View File

@ -0,0 +1 @@
export const LAST_LAYER_VERSION = 1;

View File

@ -1,6 +1,5 @@
import { fork } from 'child_process';
import { promises as fs } from 'fs';
import { tmpdir } from 'os';
import { promises as fs, existsSync } from 'fs';
import { join } from 'path';
import { v4 } from 'uuid';
@ -23,6 +22,9 @@ import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { COMMON_LAYER_NAME } from 'src/engine/integrations/serverless/drivers/constants/common-layer-name';
import { copyAndBuildDependencies } from 'src/engine/integrations/serverless/drivers/utils/copy-and-build-dependencies';
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/integrations/serverless/drivers/constants/serverless-tmpdir-folder';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
@ -39,22 +41,40 @@ export class LocalDriver
this.fileStorageService = options.fileStorageService;
}
private getInMemoryLayerFolderPath = (version: number) => {
return join(SERVERLESS_TMPDIR_FOLDER, COMMON_LAYER_NAME, `${version}`);
};
private async createLayerIfNotExists(version: number) {
const inMemoryLastVersionLayerFolderPath =
this.getInMemoryLayerFolderPath(version);
if (existsSync(inMemoryLastVersionLayerFolderPath)) {
return;
}
await copyAndBuildDependencies(inMemoryLastVersionLayerFolderPath);
}
async delete() {}
async build(serverlessFunction: ServerlessFunctionEntity) {
await this.createLayerIfNotExists(serverlessFunction.layerVersion);
const javascriptCode = await this.getCompiledCode(
serverlessFunction,
this.fileStorageService,
);
const draftFolderPath = getServerlessFolder({
serverlessFunction,
version: 'draft',
});
await this.fileStorageService.write({
file: javascriptCode,
name: BUILD_FILE_NAME,
mimeType: undefined,
folder: getServerlessFolder({
serverlessFunction,
version: 'draft',
}),
folder: draftFolderPath,
});
}
@ -68,9 +88,11 @@ export class LocalDriver
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined = undefined,
payload: object,
version: string,
): Promise<ServerlessExecuteResult> {
await this.createLayerIfNotExists(serverlessFunction.layerVersion);
const startTime = Date.now();
let fileContent = '';
@ -94,7 +116,15 @@ export class LocalDriver
throw error;
}
const tmpFilePath = join(tmpdir(), `${v4()}.js`);
const tmpFolderPath = join(SERVERLESS_TMPDIR_FOLDER, v4());
const tmpFilePath = join(tmpFolderPath, 'index.js');
await fs.symlink(
this.getInMemoryLayerFolderPath(serverlessFunction.layerVersion),
tmpFolderPath,
'dir',
);
const modifiedContent = `
process.on('message', async (message) => {

View File

@ -0,0 +1,40 @@
import { statSync, promises as fs } from 'fs';
import { promisify } from 'util';
import { exec } from 'child_process';
import { join } from 'path';
import { getLayerDependenciesDirName } from 'src/engine/integrations/serverless/drivers/utils/get-layer-dependencies-dir-name';
const execPromise = promisify(exec);
export const copyAndBuildDependencies = async (buildDirectory: string) => {
await fs.mkdir(buildDirectory, {
recursive: true,
});
await fs.cp(getLayerDependenciesDirName('latest'), buildDirectory, {
recursive: true,
});
await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
recursive: true,
});
try {
await execPromise('yarn', { cwd: buildDirectory });
} catch (error: any) {
throw new Error(error.stdout);
}
const objects = await fs.readdir(buildDirectory);
objects.forEach((object) => {
const fullPath = join(buildDirectory, object);
if (object === 'node_modules') return;
if (statSync(fullPath).isDirectory()) {
fs.rm(fullPath, { recursive: true, force: true });
} else {
fs.rm(fullPath);
}
});
};

View File

@ -0,0 +1,24 @@
import fs from 'fs/promises';
import { join } from 'path';
import { getLayerDependenciesDirName } from 'src/engine/integrations/serverless/drivers/utils/get-layer-dependencies-dir-name';
export type LayerDependencies = {
packageJson: { dependencies: object };
yarnLock: string;
};
export const getLastLayerDependencies =
async (): Promise<LayerDependencies> => {
const lastVersionLayerDirName = getLayerDependenciesDirName('latest');
const packageJson = await fs.readFile(
join(lastVersionLayerDirName, 'package.json'),
'utf8',
);
const yarnLock = await fs.readFile(
join(lastVersionLayerDirName, 'yarn.lock'),
'utf8',
);
return { packageJson: JSON.parse(packageJson), yarnLock };
};

View File

@ -0,0 +1,12 @@
import path from 'path';
import { LAST_LAYER_VERSION } from 'src/engine/integrations/serverless/drivers/layers/last-layer-version';
// Can only be used in src/engine/integrations/serverless/drivers/utils folder
export const getLayerDependenciesDirName = (
version: 'latest' | 'engine' | number,
): string => {
const formattedVersion = version === 'latest' ? LAST_LAYER_VERSION : version;
return path.resolve(__dirname, `../layers/${formattedVersion}`);
};

View File

@ -1,20 +1,22 @@
import { Injectable } from '@nestjs/common';
import { join } from 'path';
import { tmpdir } from 'os';
import fs from 'fs';
import * as fs from 'fs/promises';
import fsExtra from 'fs-extra';
import { v4 } from 'uuid';
const TEMPORARY_LAMBDA_FOLDER = 'twenty-build-lambda-temp-folder';
import { SERVERLESS_TMPDIR_FOLDER } from 'src/engine/integrations/serverless/drivers/constants/serverless-tmpdir-folder';
export const NODE_LAYER_SUBFOLDER = 'nodejs';
const TEMPORARY_LAMBDA_FOLDER = 'lambda-build';
const TEMPORARY_LAMBDA_SOURCE_FOLDER = 'src';
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
const LAMBDA_ENTRY_FILE_NAME = 'index.js';
@Injectable()
export class BuildDirectoryManagerService {
private temporaryDir = join(tmpdir(), `${TEMPORARY_LAMBDA_FOLDER}_${v4()}`);
export class LambdaBuildDirectoryManager {
private temporaryDir = join(
SERVERLESS_TMPDIR_FOLDER,
`${TEMPORARY_LAMBDA_FOLDER}-${v4()}`,
);
private lambdaHandler = `${LAMBDA_ENTRY_FILE_NAME.split('.')[0]}.handler`;
async init() {
@ -25,13 +27,7 @@ export class BuildDirectoryManagerService {
const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
const javascriptFilePath = join(sourceTemporaryDir, LAMBDA_ENTRY_FILE_NAME);
if (!fs.existsSync(this.temporaryDir)) {
await fs.promises.mkdir(this.temporaryDir);
await fs.promises.mkdir(sourceTemporaryDir);
} else {
await fsExtra.emptyDir(this.temporaryDir);
await fs.promises.mkdir(sourceTemporaryDir);
}
await fs.mkdir(sourceTemporaryDir, { recursive: true });
return {
sourceTemporaryDir,
@ -42,7 +38,6 @@ export class BuildDirectoryManagerService {
}
async clean() {
await fsExtra.emptyDir(this.temporaryDir);
await fs.promises.rmdir(this.temporaryDir);
await fs.rm(this.temporaryDir, { recursive: true, force: true });
}
}

View File

@ -2,7 +2,6 @@ import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
import {
ServerlessDriverType,
ServerlessModuleOptions,
@ -11,7 +10,6 @@ import {
export const serverlessModuleFactory = async (
environmentService: EnvironmentService,
fileStorageService: FileStorageService,
buildDirectoryManagerService: BuildDirectoryManagerService,
): Promise<ServerlessModuleOptions> => {
const driverType = environmentService.get('SERVERLESS_TYPE');
const options = { fileStorageService };
@ -37,7 +35,6 @@ export const serverlessModuleFactory = async (
type: ServerlessDriverType.Lambda,
options: {
...options,
buildDirectoryManagerService,
credentials: accessKeyId
? {
accessKeyId,

View File

@ -2,7 +2,6 @@ import { DynamicModule, Global } from '@nestjs/common';
import { LambdaDriver } from 'src/engine/integrations/serverless/drivers/lambda.driver';
import { LocalDriver } from 'src/engine/integrations/serverless/drivers/local.driver';
import { BuildDirectoryManagerService } from 'src/engine/integrations/serverless/drivers/services/build-directory-manager.service';
import { SERVERLESS_DRIVER } from 'src/engine/integrations/serverless/serverless.constants';
import {
ServerlessDriverType,
@ -28,7 +27,7 @@ export class ServerlessModule {
return {
module: ServerlessModule,
imports: options.imports || [],
providers: [ServerlessService, BuildDirectoryManagerService, provider],
providers: [ServerlessService, provider],
exports: [ServerlessService],
};
}

View File

@ -29,7 +29,7 @@ export class ServerlessService implements ServerlessDriver {
async execute(
serverlessFunction: ServerlessFunctionEntity,
payload: object | undefined = undefined,
payload: object,
version: string,
): Promise<ServerlessExecuteResult> {
return this.driver.execute(serverlessFunction, payload, version);

View File

@ -1,6 +1,6 @@
import { Field, InputType } from '@nestjs/graphql';
import { IsNotEmpty, IsObject, IsOptional, IsUUID } from 'class-validator';
import { IsNotEmpty, IsObject, IsUUID } from 'class-validator';
import graphqlTypeJson from 'graphql-type-json';
import { UUIDScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@ -16,11 +16,9 @@ export class ExecuteServerlessFunctionInput {
@Field(() => graphqlTypeJson, {
description: 'Payload in JSON format',
nullable: true,
})
@IsObject()
@IsOptional()
payload?: JSON;
payload: JSON;
@Field(() => String, {
nullable: false,

View File

@ -43,7 +43,6 @@ export class ServerlessFunctionDTO {
id: string;
@IsString()
@IsNotEmpty()
@Field()
name: string;

View File

@ -14,7 +14,6 @@ export class UpdateServerlessFunctionInput {
id: string;
@IsString()
@IsNotEmpty()
@Field()
name: string;

View File

@ -35,6 +35,9 @@ export class ServerlessFunctionEntity {
@Column({ nullable: false, default: ServerlessFunctionRuntime.NODE18 })
runtime: ServerlessFunctionRuntime;
@Column({ nullable: true })
layerVersion: number;
@Column({
nullable: false,
default: ServerlessFunctionSyncStatus.NOT_READY,

View File

@ -4,6 +4,7 @@ import { InjectRepository } from '@nestjs/typeorm';
import { FileUpload, GraphQLUpload } from 'graphql-upload';
import { Repository } from 'typeorm';
import graphqlTypeJson from 'graphql-type-json';
import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
@ -51,7 +52,18 @@ export class ServerlessFunctionResolver {
}
}
@Query(() => String)
@Query(() => graphqlTypeJson)
async getAvailablePackages(@AuthWorkspace() { id: workspaceId }: Workspace) {
try {
await this.checkFeatureFlag(workspaceId);
return await this.serverlessFunctionService.getAvailablePackages();
} catch (error) {
serverlessFunctionGraphQLApiExceptionHandler(error);
}
}
@Query(() => String, { nullable: true })
async getServerlessFunctionSourceCode(
@Args('input') input: GetServerlessFunctionSourceCodeInput,
@AuthWorkspace() { id: workspaceId }: Workspace,

View File

@ -27,6 +27,8 @@ import {
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { serverlessFunctionCreateHash } from 'src/engine/metadata-modules/serverless-function/utils/serverless-function-create-hash.utils';
import { isDefined } from 'src/utils/is-defined';
import { getLastLayerDependencies } from 'src/engine/integrations/serverless/drivers/utils/get-last-layer-dependencies';
import { LAST_LAYER_VERSION } from 'src/engine/integrations/serverless/drivers/layers/last-layer-version';
@Injectable()
export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFunctionEntity> {
@ -46,22 +48,21 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
id: string,
version: string,
) {
const serverlessFunction = await this.serverlessFunctionRepository.findOne({
where: {
id,
workspaceId,
},
});
if (!serverlessFunction) {
throw new ServerlessFunctionException(
`Function does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
try {
const serverlessFunction =
await this.serverlessFunctionRepository.findOne({
where: {
id,
workspaceId,
},
});
if (!serverlessFunction) {
throw new ServerlessFunctionException(
`Function does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
}
const folderPath = getServerlessFolder({
serverlessFunction,
version,
@ -75,10 +76,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
return await readFileContent(fileStream);
} catch (error) {
if (error.code === FileStorageExceptionCode.FILE_NOT_FOUND) {
throw new ServerlessFunctionException(
`Function Version '${version}' does not exist`,
ServerlessFunctionExceptionCode.SERVERLESS_FUNCTION_NOT_FOUND,
);
return;
}
throw error;
}
@ -87,7 +85,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
async executeOneServerlessFunction(
id: string,
workspaceId: string,
payload: object | undefined = undefined,
payload: object,
version = 'latest',
): Promise<ServerlessExecuteResult> {
await this.throttleExecution(workspaceId);
@ -106,15 +104,6 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
);
}
if (
functionToExecute.syncStatus === ServerlessFunctionSyncStatus.NOT_READY
) {
await this.serverlessService.build(functionToExecute, version);
await super.updateOne(functionToExecute.id, {
syncStatus: ServerlessFunctionSyncStatus.READY,
});
}
return this.serverlessService.execute(functionToExecute, payload, version);
}
@ -144,8 +133,8 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
);
if (
serverlessFunctionCreateHash(latestCode) ===
serverlessFunctionCreateHash(draftCode)
serverlessFunctionCreateHash(latestCode || '') ===
serverlessFunctionCreateHash(draftCode || '')
) {
throw new Error(
'Cannot publish a new version when code has not changed',
@ -224,6 +213,9 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
name: serverlessFunctionInput.name,
description: serverlessFunctionInput.description,
syncStatus: ServerlessFunctionSyncStatus.NOT_READY,
sourceCodeHash: serverlessFunctionCreateHash(
serverlessFunctionInput.code,
),
});
const fileFolder = getServerlessFolder({
@ -238,9 +230,34 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
folder: fileFolder,
});
await this.serverlessService.build(existingServerlessFunction, 'draft');
await super.updateOne(existingServerlessFunction.id, {
syncStatus: ServerlessFunctionSyncStatus.READY,
});
return await this.findById(existingServerlessFunction.id);
}
async getAvailablePackages() {
const { packageJson, yarnLock } = await getLastLayerDependencies();
const packageVersionRegex = /^"([^@]+)@.*?":\n\s+version: (.+)$/gm;
const versions: Record<string, string> = {};
let match: RegExpExecArray | null;
while ((match = packageVersionRegex.exec(yarnLock)) !== null) {
const packageName = match[1].split('@', 1)[0];
const version = match[2];
if (packageJson.dependencies[packageName]) {
versions[packageName] = version;
}
}
return versions;
}
async createOneServerlessFunction(
serverlessFunctionInput: CreateServerlessFunctionFromFileInput,
code: FileUpload | string,
@ -258,6 +275,7 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
...serverlessFunctionInput,
workspaceId,
sourceCodeHash: serverlessFunctionCreateHash(typescriptCode),
layerVersion: LAST_LAYER_VERSION,
});
const draftFileFolder = getServerlessFolder({
@ -272,6 +290,8 @@ export class ServerlessFunctionService extends TypeOrmQueryService<ServerlessFun
folder: draftFileFolder,
});
await this.serverlessService.build(createdServerlessFunction, 'draft');
return await this.findById(createdServerlessFunction.id);
}

View File

@ -37,7 +37,7 @@ export class CodeActionExecutor implements WorkflowStepExecutor {
await this.serverlessFunctionService.executeOneServerlessFunction(
step.settings.serverlessFunctionId,
workspaceId,
payload,
payload || {},
);
return { data: result.data, ...(result.error && { error: result.error }) };