Refactor backend folder structure (#4505)

* Refactor backend folder structure

Co-authored-by: Charles Bochet <charles@twenty.com>

* fix tests

* fix

* move yoga hooks

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
Weiko
2024-03-15 18:37:09 +01:00
committed by GitHub
parent afb9b3e375
commit 2c09096edd
523 changed files with 1386 additions and 1856 deletions

View File

@ -0,0 +1,78 @@
import { Cache } from '@nestjs/cache-manager';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
const cacheStorageNamespace = CacheStorageNamespace.Messaging;
describe('CacheStorageService', () => {
let cacheStorageService: CacheStorageService;
let cacheManagerMock: Partial<Cache>;
beforeEach(() => {
cacheManagerMock = {
get: jest.fn(),
set: jest.fn(),
};
cacheStorageService = new CacheStorageService(
cacheManagerMock as Cache,
cacheStorageNamespace,
);
});
afterEach(() => {
jest.clearAllMocks();
});
describe('get', () => {
it('should call cacheManager.get with the correct namespaced key', async () => {
const key = 'testKey';
const namespacedKey = `${cacheStorageNamespace}:${key}`;
await cacheStorageService.get(key);
expect(cacheManagerMock.get).toHaveBeenCalledWith(namespacedKey);
});
it('should return the value returned by cacheManager.get', async () => {
const key = 'testKey';
const value = 'testValue';
jest.spyOn(cacheManagerMock, 'get').mockResolvedValue(value);
const result = await cacheStorageService.get(key);
expect(result).toBe(value);
});
});
describe('set', () => {
it('should call cacheManager.set with the correct namespaced key, value, and optional ttl', async () => {
const key = 'testKey';
const value = 'testValue';
const ttl = 60;
const namespacedKey = `${cacheStorageNamespace}:${key}`;
await cacheStorageService.set(key, value, ttl);
expect(cacheManagerMock.set).toHaveBeenCalledWith(
namespacedKey,
value,
ttl,
);
});
it('should not throw if cacheManager.set resolves successfully', async () => {
const key = 'testKey';
const value = 'testValue';
const ttl = 60;
jest.spyOn(cacheManagerMock, 'set').mockResolvedValue(undefined);
await expect(
cacheStorageService.set(key, value, ttl),
).resolves.not.toThrow();
});
});
});

View File

@ -0,0 +1,46 @@
import { CacheModuleOptions } from '@nestjs/common';
import { redisStore } from 'cache-manager-redis-yet';
import { CacheStorageType } from 'src/engine/integrations/cache-storage/types/cache-storage-type.enum';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
export const cacheStorageModuleFactory = (
environmentService: EnvironmentService,
): CacheModuleOptions => {
const cacheStorageType = environmentService.get('CACHE_STORAGE_TYPE');
const cacheStorageTtl = environmentService.get('CACHE_STORAGE_TTL');
const cacheModuleOptions: CacheModuleOptions = {
isGlobal: true,
ttl: cacheStorageTtl * 1000,
};
switch (cacheStorageType) {
case CacheStorageType.Memory: {
return cacheModuleOptions;
}
case CacheStorageType.Redis: {
const host = environmentService.get('REDIS_HOST');
const port = environmentService.get('REDIS_PORT');
if (!(host && port)) {
throw new Error(
`${cacheStorageType} cache storage requires host: ${host} and port: ${port} to be defined, check your .env file`,
);
}
return {
...cacheModuleOptions,
store: redisStore,
socket: {
host,
port,
},
};
}
default:
throw new Error(
`Invalid cache-storage (${cacheStorageType}), check your .env file`,
);
}
};

View File

@ -0,0 +1,31 @@
import { Module, Global } from '@nestjs/common';
import { CacheModule, CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
import { ConfigModule } from '@nestjs/config';
import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { cacheStorageModuleFactory } from 'src/engine/integrations/cache-storage/cache-storage.module-factory';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
@Global()
@Module({
imports: [
CacheModule.registerAsync({
isGlobal: true,
imports: [ConfigModule],
useFactory: cacheStorageModuleFactory,
inject: [EnvironmentService],
}),
],
providers: [
...Object.values(CacheStorageNamespace).map((cacheStorageNamespace) => ({
provide: cacheStorageNamespace,
useFactory: (cacheManager: Cache) => {
return new CacheStorageService(cacheManager, cacheStorageNamespace);
},
inject: [CACHE_MANAGER],
})),
],
exports: [...Object.values(CacheStorageNamespace)],
})
export class CacheStorageModule {}

View File

@ -0,0 +1,25 @@
import { Inject, Injectable } from '@nestjs/common';
import { CACHE_MANAGER, Cache } from '@nestjs/cache-manager';
import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum';
@Injectable()
export class CacheStorageService {
constructor(
@Inject(CACHE_MANAGER)
private readonly cacheManager: Cache,
private readonly namespace: CacheStorageNamespace,
) {}
async get<T>(key: string): Promise<T | undefined> {
return this.cacheManager.get(`${this.namespace}:${key}`);
}
async set<T>(key: string, value: T, ttl?: number) {
return this.cacheManager.set(`${this.namespace}:${key}`, value, ttl);
}
async del(key: string) {
return this.cacheManager.del(`${this.namespace}:${key}`);
}
}

View File

@ -0,0 +1,4 @@
export enum CacheStorageNamespace {
Messaging = 'messaging',
WorkspaceSchema = 'workspaceSchema',
}

View File

@ -0,0 +1,4 @@
export enum CacheStorageType {
Memory = 'memory',
Redis = 'redis',
}

View File

@ -0,0 +1,5 @@
import { SendMailOptions } from 'nodemailer';
export interface EmailDriver {
send(sendMailOptions: SendMailOptions): Promise<void>;
}

View File

@ -0,0 +1,20 @@
import { Logger } from '@nestjs/common';
import { SendMailOptions } from 'nodemailer';
import { EmailDriver } from 'src/engine/integrations/email/drivers/interfaces/email-driver.interface';
export class LoggerDriver implements EmailDriver {
private readonly logger = new Logger(LoggerDriver.name);
async send(sendMailOptions: SendMailOptions): Promise<void> {
const info =
`Sent email to: ${sendMailOptions.to}\n` +
`From: ${sendMailOptions.from}\n` +
`Subject: ${sendMailOptions.subject}\n` +
`Content Text: ${sendMailOptions.text}\n` +
`Content HTML: ${sendMailOptions.html}`;
this.logger.log(info);
}
}

View File

@ -0,0 +1,26 @@
import { Logger } from '@nestjs/common';
import { createTransport, Transporter, SendMailOptions } from 'nodemailer';
import SMTPConnection from 'nodemailer/lib/smtp-connection';
import { EmailDriver } from 'src/engine/integrations/email/drivers/interfaces/email-driver.interface';
export class SmtpDriver implements EmailDriver {
private readonly logger = new Logger(SmtpDriver.name);
private transport: Transporter;
constructor(options: SMTPConnection.Options) {
this.transport = createTransport(options);
}
async send(sendMailOptions: SendMailOptions): Promise<void> {
this.transport
.sendMail(sendMailOptions)
.then(() =>
this.logger.log(`Email to '${sendMailOptions.to}' successfully sent`),
)
.catch((err) =>
this.logger.error(`sending email to '${sendMailOptions.to}': ${err}`),
);
}
}

View File

@ -0,0 +1,16 @@
import { Injectable } from '@nestjs/common';
import { SendMailOptions } from 'nodemailer';
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { EmailSenderService } from 'src/engine/integrations/email/email-sender.service';
@Injectable()
export class EmailSenderJob implements MessageQueueJob<SendMailOptions> {
constructor(private readonly emailSenderService: EmailSenderService) {}
async handle(data: SendMailOptions): Promise<void> {
await this.emailSenderService.send(data);
}
}

View File

@ -0,0 +1,16 @@
import { Inject, Injectable } from '@nestjs/common';
import { SendMailOptions } from 'nodemailer';
import { EmailDriver } from 'src/engine/integrations/email/drivers/interfaces/email-driver.interface';
import { EMAIL_DRIVER } from 'src/engine/integrations/email/email.constants';
@Injectable()
export class EmailSenderService implements EmailDriver {
constructor(@Inject(EMAIL_DRIVER) private driver: EmailDriver) {}
async send(sendMailOptions: SendMailOptions): Promise<void> {
await this.driver.send(sendMailOptions);
}
}

View File

@ -0,0 +1 @@
export const EMAIL_DRIVER = Symbol('EMAIL_DRIVER');

View File

@ -0,0 +1,40 @@
import {
EmailDriver,
EmailModuleOptions,
} from 'src/engine/integrations/email/interfaces/email.interface';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
export const emailModuleFactory = (
environmentService: EnvironmentService,
): EmailModuleOptions => {
const driver = environmentService.get('EMAIL_DRIVER');
switch (driver) {
case EmailDriver.Logger: {
return;
}
case EmailDriver.Smtp: {
const host = environmentService.get('EMAIL_SMTP_HOST');
const port = environmentService.get('EMAIL_SMTP_PORT');
const user = environmentService.get('EMAIL_SMTP_USER');
const pass = environmentService.get('EMAIL_SMTP_PASSWORD');
if (!(host && port)) {
throw new Error(
`${driver} email driver requires host: ${host} and port: ${port} to be defined, check your .env file`,
);
}
const auth = user && pass ? { user, pass } : undefined;
if (auth) {
return { host, port, auth };
}
return { host, port };
}
default:
throw new Error(`Invalid email driver (${driver}), check your .env file`);
}
};

View File

@ -0,0 +1,30 @@
import { DynamicModule, Global } from '@nestjs/common';
import { EmailModuleAsyncOptions } from 'src/engine/integrations/email/interfaces/email.interface';
import { EMAIL_DRIVER } from 'src/engine/integrations/email/email.constants';
import { LoggerDriver } from 'src/engine/integrations/email/drivers/logger.driver';
import { SmtpDriver } from 'src/engine/integrations/email/drivers/smtp.driver';
import { EmailService } from 'src/engine/integrations/email/email.service';
import { EmailSenderService } from 'src/engine/integrations/email/email-sender.service';
@Global()
export class EmailModule {
static forRoot(options: EmailModuleAsyncOptions): DynamicModule {
const provider = {
provide: EMAIL_DRIVER,
useFactory: (...args: any[]) => {
const config = options.useFactory(...args);
return config ? new SmtpDriver(config) : new LoggerDriver();
},
inject: options.inject || [],
};
return {
module: EmailModule,
providers: [EmailSenderService, EmailService, provider],
exports: [EmailSenderService, EmailService],
};
}
}

View File

@ -0,0 +1,23 @@
import { Inject, Injectable } from '@nestjs/common';
import { SendMailOptions } from 'nodemailer';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { EmailSenderJob } from 'src/engine/integrations/email/email-sender.job';
@Injectable()
export class EmailService {
constructor(
@Inject(MessageQueue.emailQueue)
private readonly messageQueueService: MessageQueueService,
) {}
async send(sendMailOptions: SendMailOptions): Promise<void> {
await this.messageQueueService.add<SendMailOptions>(
EmailSenderJob.name,
sendMailOptions,
{ retryLimit: 3 },
);
}
}

View File

@ -0,0 +1,15 @@
import { FactoryProvider, ModuleMetadata } from '@nestjs/common';
import SMTPConnection from 'nodemailer/lib/smtp-connection';
export enum EmailDriver {
Logger = 'logger',
Smtp = 'smtp',
}
export type EmailModuleOptions = SMTPConnection.Options | undefined;
export type EmailModuleAsyncOptions = {
useFactory: (...args: any[]) => EmailModuleOptions;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;

View File

@ -0,0 +1,66 @@
import { plainToClass } from 'class-transformer';
import { CastToLogLevelArray } from 'src/engine/integrations/environment/decorators/cast-to-log-level-array.decorator';
class TestClass {
@CastToLogLevelArray()
logLevels?: any;
}
describe('CastToLogLevelArray Decorator', () => {
it('should cast "log" to ["log"]', () => {
const transformedClass = plainToClass(TestClass, { logLevels: 'log' });
expect(transformedClass.logLevels).toStrictEqual(['log']);
});
it('should cast "error" to ["error"]', () => {
const transformedClass = plainToClass(TestClass, { logLevels: 'error' });
expect(transformedClass.logLevels).toStrictEqual(['error']);
});
it('should cast "warn" to ["warn"]', () => {
const transformedClass = plainToClass(TestClass, { logLevels: 'warn' });
expect(transformedClass.logLevels).toStrictEqual(['warn']);
});
it('should cast "debug" to ["debug"]', () => {
const transformedClass = plainToClass(TestClass, { logLevels: 'debug' });
expect(transformedClass.logLevels).toStrictEqual(['debug']);
});
it('should cast "verbose" to ["verbose"]', () => {
const transformedClass = plainToClass(TestClass, { logLevels: 'verbose' });
expect(transformedClass.logLevels).toStrictEqual(['verbose']);
});
it('should cast "verbose,error,warn" to ["verbose", "error", "warn"]', () => {
const transformedClass = plainToClass(TestClass, {
logLevels: 'verbose,error,warn',
});
expect(transformedClass.logLevels).toStrictEqual([
'verbose',
'error',
'warn',
]);
});
it('should cast "toto" to undefined', () => {
const transformedClass = plainToClass(TestClass, { logLevels: 'toto' });
expect(transformedClass.logLevels).toBeUndefined();
});
it('should cast "verbose,error,toto" to undefined', () => {
const transformedClass = plainToClass(TestClass, {
logLevels: 'verbose,error,toto',
});
expect(transformedClass.logLevels).toBeUndefined();
});
});

View File

@ -0,0 +1,58 @@
import { plainToClass } from 'class-transformer';
import { CastToPositiveNumber } from 'src/engine/integrations/environment/decorators/cast-to-positive-number.decorator';
class TestClass {
@CastToPositiveNumber()
numberProperty?: any;
}
describe('CastToPositiveNumber Decorator', () => {
it('should cast number to number', () => {
const transformedClass = plainToClass(TestClass, { numberProperty: 123 });
expect(transformedClass.numberProperty).toBe(123);
});
it('should cast string to number', () => {
const transformedClass = plainToClass(TestClass, { numberProperty: '123' });
expect(transformedClass.numberProperty).toBe(123);
});
it('should cast null to undefined', () => {
const transformedClass = plainToClass(TestClass, { numberProperty: null });
expect(transformedClass.numberProperty).toBe(undefined);
});
it('should cast negative number to undefined', () => {
const transformedClass = plainToClass(TestClass, { numberProperty: -12 });
expect(transformedClass.numberProperty).toBe(undefined);
});
it('should cast undefined to undefined', () => {
const transformedClass = plainToClass(TestClass, {
numberProperty: undefined,
});
expect(transformedClass.numberProperty).toBe(undefined);
});
it('should cast NaN string to undefined', () => {
const transformedClass = plainToClass(TestClass, {
numberProperty: 'toto',
});
expect(transformedClass.numberProperty).toBe(undefined);
});
it('should cast a negative string to undefined', () => {
const transformedClass = plainToClass(TestClass, {
numberProperty: '-123',
});
expect(transformedClass.numberProperty).toBe(undefined);
});
});

View File

@ -0,0 +1,18 @@
import { Transform } from 'class-transformer';
export const CastToBoolean = () =>
Transform(({ value }: { value: string }) => toBoolean(value));
const toBoolean = (value: any) => {
if (typeof value === 'boolean') {
return value;
}
if (['true', 'on', 'yes', '1'].includes(value.toLowerCase())) {
return true;
}
if (['false', 'off', 'no', '0'].includes(value.toLowerCase())) {
return false;
}
return undefined;
};

View File

@ -0,0 +1,19 @@
import { Transform } from 'class-transformer';
export const CastToLogLevelArray = () =>
Transform(({ value }: { value: string }) => toLogLevelArray(value));
const toLogLevelArray = (value: any) => {
if (typeof value === 'string') {
const rawLogLevels = value.split(',').map((level) => level.trim());
const isInvalid = rawLogLevels.some(
(level) => !['log', 'error', 'warn', 'debug', 'verbose'].includes(level),
);
if (!isInvalid) {
return rawLogLevels;
}
}
return undefined;
};

View File

@ -0,0 +1,15 @@
import { Transform } from 'class-transformer';
export const CastToPositiveNumber = () =>
Transform(({ value }: { value: string }) => toNumber(value));
const toNumber = (value: any) => {
if (typeof value === 'number') {
return value >= 0 ? value : undefined;
}
if (typeof value === 'string') {
return isNaN(+value) ? undefined : toNumber(+value);
}
return undefined;
};

View File

@ -0,0 +1,12 @@
import { Transform } from 'class-transformer';
export const CastToStringArray = () =>
Transform(({ value }: { value: string }) => toStringArray(value));
const toStringArray = (value: any) => {
if (typeof value === 'string') {
return value.split(',').map((item) => item.trim());
}
return undefined;
};

View File

@ -0,0 +1,27 @@
import {
registerDecorator,
ValidationOptions,
ValidatorConstraint,
ValidatorConstraintInterface,
} from 'class-validator';
@ValidatorConstraint({ async: true })
export class IsAWSRegionConstraint implements ValidatorConstraintInterface {
validate(region: string) {
const regex = /^[a-z]{2}-[a-z]+-\d{1}$/;
return regex.test(region); // Returns true if region matches regex
}
}
export const IsAWSRegion =
(validationOptions?: ValidationOptions) =>
(object: object, propertyName: string) => {
registerDecorator({
target: object.constructor,
propertyName: propertyName,
options: validationOptions,
constraints: [],
validator: IsAWSRegionConstraint,
});
};

View File

@ -0,0 +1,28 @@
import {
registerDecorator,
ValidationOptions,
ValidatorConstraint,
ValidatorConstraintInterface,
} from 'class-validator';
@ValidatorConstraint({ async: true })
export class IsDurationConstraint implements ValidatorConstraintInterface {
validate(duration: string) {
const regex =
/^-?[0-9]+(.[0-9]+)?(m(illiseconds?)?|s(econds?)?|h((ou)?rs?)?|d(ays?)?|w(eeks?)?|M(onths?)?|y(ears?)?)?$/;
return regex.test(duration); // Returns true if duration matches regex
}
}
export const IsDuration =
(validationOptions?: ValidationOptions) =>
(object: object, propertyName: string) => {
registerDecorator({
target: object.constructor,
propertyName: propertyName,
options: validationOptions,
constraints: [],
validator: IsDurationConstraint,
});
};

View File

@ -0,0 +1,32 @@
import {
registerDecorator,
ValidationArguments,
ValidationOptions,
} from 'class-validator';
export const IsStrictlyLowerThan = (
property: string,
validationOptions?: ValidationOptions,
) => {
return (object: object, propertyName: string) => {
registerDecorator({
name: 'isStrictlyLowerThan',
target: object.constructor,
propertyName: propertyName,
constraints: [property],
options: validationOptions,
validator: {
validate(value: any, args: ValidationArguments) {
const [relatedPropertyName] = args.constraints;
const relatedValue = (args.object as any)[relatedPropertyName];
return (
typeof value === 'number' &&
typeof relatedValue === 'number' &&
value < relatedValue
);
},
},
});
};
};

View File

@ -0,0 +1,289 @@
import { LogLevel } from '@nestjs/common';
import { plainToClass } from 'class-transformer';
import {
IsEnum,
IsOptional,
IsString,
IsUrl,
ValidateIf,
validateSync,
IsBoolean,
IsNumber,
IsDefined,
} from 'class-validator';
import { EmailDriver } from 'src/engine/integrations/email/interfaces/email.interface';
import { assert } from 'src/utils/assert';
import { CastToStringArray } from 'src/engine/integrations/environment/decorators/cast-to-string-array.decorator';
import { ExceptionHandlerDriver } from 'src/engine/integrations/exception-handler/interfaces';
import { StorageDriverType } from 'src/engine/integrations/file-storage/interfaces';
import { LoggerDriverType } from 'src/engine/integrations/logger/interfaces';
import { IsStrictlyLowerThan } from 'src/engine/integrations/environment/decorators/is-strictly-lower-than.decorator';
import { IsDuration } from './decorators/is-duration.decorator';
import { AwsRegion } from './interfaces/aws-region.interface';
import { IsAWSRegion } from './decorators/is-aws-region.decorator';
import { CastToBoolean } from './decorators/cast-to-boolean.decorator';
import { SupportDriver } from './interfaces/support.interface';
import { CastToPositiveNumber } from './decorators/cast-to-positive-number.decorator';
import { CastToLogLevelArray } from './decorators/cast-to-log-level-array.decorator';
export class EnvironmentVariables {
// Misc
@CastToBoolean()
@IsOptional()
@IsBoolean()
DEBUG_MODE: boolean;
@CastToBoolean()
@IsOptional()
@IsBoolean()
SIGN_IN_PREFILLED: boolean;
@CastToBoolean()
@IsOptional()
@IsBoolean()
IS_BILLING_ENABLED: boolean;
@IsString()
@ValidateIf((env) => env.IS_BILLING_ENABLED === true)
BILLING_PLAN_REQUIRED_LINK: string;
@IsString()
@ValidateIf((env) => env.IS_BILLING_ENABLED === true)
BILLING_STRIPE_BASE_PLAN_PRODUCT_ID: string;
@IsNumber()
@CastToPositiveNumber()
@IsOptional()
@ValidateIf((env) => env.IS_BILLING_ENABLED === true)
BILLING_FREE_TRIAL_DURATION_IN_DAYS: number;
@IsString()
@ValidateIf((env) => env.IS_BILLING_ENABLED === true)
BILLING_STRIPE_API_KEY: string;
@IsString()
@ValidateIf((env) => env.IS_BILLING_ENABLED === true)
BILLING_STRIPE_WEBHOOK_SECRET: string;
@CastToBoolean()
@IsOptional()
@IsBoolean()
TELEMETRY_ENABLED: boolean;
@CastToBoolean()
@IsOptional()
@IsBoolean()
TELEMETRY_ANONYMIZATION_ENABLED: boolean;
@CastToPositiveNumber()
@IsNumber()
@IsOptional()
PORT: number;
// Database
@IsDefined()
@IsUrl({
protocols: ['postgres'],
require_tld: false,
allow_underscores: true,
})
PG_DATABASE_URL: string;
// Frontend URL
@IsUrl({ require_tld: false })
FRONT_BASE_URL: string;
// Server URL
@IsUrl({ require_tld: false })
@IsOptional()
SERVER_URL: string;
// Json Web Token
@IsString()
ACCESS_TOKEN_SECRET: string;
@IsDuration()
@IsOptional()
ACCESS_TOKEN_EXPIRES_IN: string;
@IsString()
REFRESH_TOKEN_SECRET: string;
@IsDuration()
@IsOptional()
REFRESH_TOKEN_EXPIRES_IN: string;
@IsDuration()
@IsOptional()
REFRESH_TOKEN_COOL_DOWN: string;
@IsString()
LOGIN_TOKEN_SECRET: string;
@IsDuration()
@IsOptional()
LOGIN_TOKEN_EXPIRES_IN: string;
// Auth
@IsUrl({ require_tld: false })
@IsOptional()
FRONT_AUTH_CALLBACK_URL: string;
@CastToBoolean()
@IsOptional()
@IsBoolean()
AUTH_GOOGLE_ENABLED: boolean;
@IsString()
@ValidateIf((env) => env.AUTH_GOOGLE_ENABLED === true)
AUTH_GOOGLE_CLIENT_ID: string;
@IsString()
@ValidateIf((env) => env.AUTH_GOOGLE_ENABLED === true)
AUTH_GOOGLE_CLIENT_SECRET: string;
@IsUrl({ require_tld: false })
@ValidateIf((env) => env.AUTH_GOOGLE_ENABLED === true)
AUTH_GOOGLE_CALLBACK_URL: string;
// Storage
@IsEnum(StorageDriverType)
@IsOptional()
STORAGE_TYPE: StorageDriverType;
@ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3)
@IsAWSRegion()
STORAGE_S3_REGION: AwsRegion;
@ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3)
@IsString()
STORAGE_S3_NAME: string;
@ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3)
@IsString()
STORAGE_S3_ENDPOINT: string;
@IsString()
@ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.Local)
STORAGE_LOCAL_PATH: string;
// Support
@IsEnum(SupportDriver)
@IsOptional()
SUPPORT_DRIVER: SupportDriver;
@ValidateIf((env) => env.SUPPORT_DRIVER === SupportDriver.Front)
@IsString()
SUPPORT_FRONT_CHAT_ID: string;
@ValidateIf((env) => env.SUPPORT_DRIVER === SupportDriver.Front)
@IsString()
SUPPORT_FRONT_HMAC_KEY: string;
@IsEnum(LoggerDriverType)
@IsOptional()
LOGGER_DRIVER: LoggerDriverType;
@IsEnum(ExceptionHandlerDriver)
@IsOptional()
EXCEPTION_HANDLER_DRIVER: ExceptionHandlerDriver;
@CastToLogLevelArray()
@IsOptional()
LOG_LEVELS: LogLevel[];
@CastToStringArray()
@IsOptional()
DEMO_WORKSPACE_IDS: string[];
@ValidateIf(
(env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry,
)
@IsString()
SENTRY_DSN: string;
@IsDuration()
@IsOptional()
PASSWORD_RESET_TOKEN_EXPIRES_IN: string;
@CastToPositiveNumber()
@IsNumber()
@ValidateIf((env) => env.WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION > 0)
@IsStrictlyLowerThan('WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION', {
message:
'"WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION" should be strictly lower that "WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION"',
})
@ValidateIf((env) => env.WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION > 0)
WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION: number;
@CastToPositiveNumber()
@IsNumber()
@ValidateIf((env) => env.WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION > 0)
WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION: number;
@CastToBoolean()
@IsOptional()
@IsBoolean()
IS_SIGN_UP_DISABLED: boolean;
@CastToPositiveNumber()
@IsOptional()
@IsNumber()
MUTATION_MAXIMUM_RECORD_AFFECTED: number;
REDIS_HOST: string;
REDIS_PORT: number;
API_TOKEN_EXPIRES_IN: string;
SHORT_TERM_TOKEN_EXPIRES_IN: string;
MESSAGING_PROVIDER_GMAIL_ENABLED: boolean;
MESSAGING_PROVIDER_GMAIL_CALLBACK_URL: string;
MESSAGE_QUEUE_TYPE: string;
EMAIL_FROM_ADDRESS: string;
EMAIL_SYSTEM_ADDRESS: string;
EMAIL_FROM_NAME: string;
EMAIL_DRIVER: EmailDriver;
EMAIL_SMTP_HOST: string;
EMAIL_SMTP_PORT: number;
EMAIL_SMTP_USER: string;
EMAIL_SMTP_PASSWORD: string;
OPENROUTER_API_KEY: string;
API_RATE_LIMITING_TTL: number;
API_RATE_LIMITING_LIMIT: number;
CACHE_STORAGE_TYPE: string;
CACHE_STORAGE_TTL: number;
CALENDAR_PROVIDER_GOOGLE_ENABLED: boolean;
AUTH_GOOGLE_APIS_CALLBACK_URL: string;
}
export const validate = (config: Record<string, unknown>) => {
const validatedConfig = plainToClass(EnvironmentVariables, config);
const errors = validateSync(validatedConfig);
assert(!errors.length, errors.toString());
return validatedConfig;
};

View File

@ -0,0 +1,77 @@
import { EmailDriver } from 'src/engine/integrations/email/interfaces/email.interface';
import { SupportDriver } from 'src/engine/integrations/environment/interfaces/support.interface';
import { ExceptionHandlerDriver } from 'src/engine/integrations/exception-handler/interfaces';
import { StorageDriverType } from 'src/engine/integrations/file-storage/interfaces';
import { LoggerDriverType } from 'src/engine/integrations/logger/interfaces';
import { MessageQueueDriverType } from 'src/engine/integrations/message-queue/interfaces';
import { EnvironmentVariables } from 'src/engine/integrations/environment/environment-variables';
const EnvironmentDefault = new EnvironmentVariables();
EnvironmentDefault.DEBUG_MODE = false;
EnvironmentDefault.SIGN_IN_PREFILLED = false;
EnvironmentDefault.IS_BILLING_ENABLED = false;
EnvironmentDefault.BILLING_PLAN_REQUIRED_LINK = '';
EnvironmentDefault.BILLING_STRIPE_BASE_PLAN_PRODUCT_ID = '';
EnvironmentDefault.BILLING_FREE_TRIAL_DURATION_IN_DAYS = 7;
EnvironmentDefault.BILLING_STRIPE_API_KEY = '';
EnvironmentDefault.BILLING_STRIPE_WEBHOOK_SECRET = '';
EnvironmentDefault.TELEMETRY_ENABLED = true;
EnvironmentDefault.TELEMETRY_ANONYMIZATION_ENABLED = true;
EnvironmentDefault.PORT = 3000;
EnvironmentDefault.REDIS_HOST = '127.0.0.1';
EnvironmentDefault.REDIS_PORT = 6379;
EnvironmentDefault.PG_DATABASE_URL = '';
EnvironmentDefault.FRONT_BASE_URL = '';
EnvironmentDefault.SERVER_URL = '';
EnvironmentDefault.ACCESS_TOKEN_SECRET = 'random_string';
EnvironmentDefault.ACCESS_TOKEN_EXPIRES_IN = '30m';
EnvironmentDefault.REFRESH_TOKEN_SECRET = 'random_string';
EnvironmentDefault.REFRESH_TOKEN_EXPIRES_IN = '30m';
EnvironmentDefault.REFRESH_TOKEN_COOL_DOWN = '1m';
EnvironmentDefault.LOGIN_TOKEN_SECRET = 'random_string';
EnvironmentDefault.LOGIN_TOKEN_EXPIRES_IN = '30m';
EnvironmentDefault.API_TOKEN_EXPIRES_IN = '100y';
EnvironmentDefault.SHORT_TERM_TOKEN_EXPIRES_IN = '5m';
EnvironmentDefault.FRONT_AUTH_CALLBACK_URL = '';
EnvironmentDefault.MESSAGING_PROVIDER_GMAIL_ENABLED = false;
EnvironmentDefault.MESSAGING_PROVIDER_GMAIL_CALLBACK_URL = '';
EnvironmentDefault.AUTH_GOOGLE_ENABLED = false;
EnvironmentDefault.AUTH_GOOGLE_CLIENT_ID = '';
EnvironmentDefault.AUTH_GOOGLE_CLIENT_SECRET = '';
EnvironmentDefault.AUTH_GOOGLE_CALLBACK_URL = '';
EnvironmentDefault.STORAGE_TYPE = StorageDriverType.Local;
EnvironmentDefault.STORAGE_S3_REGION = 'aws-east-1';
EnvironmentDefault.STORAGE_S3_NAME = '';
EnvironmentDefault.STORAGE_S3_ENDPOINT = '';
EnvironmentDefault.STORAGE_LOCAL_PATH = '.local-storage';
EnvironmentDefault.MESSAGE_QUEUE_TYPE = MessageQueueDriverType.Sync;
EnvironmentDefault.EMAIL_FROM_ADDRESS = 'noreply@yourdomain.com';
EnvironmentDefault.EMAIL_SYSTEM_ADDRESS = 'system@yourdomain.com';
EnvironmentDefault.EMAIL_FROM_NAME = 'John from Twenty';
EnvironmentDefault.EMAIL_DRIVER = EmailDriver.Logger;
EnvironmentDefault.EMAIL_SMTP_HOST = '';
EnvironmentDefault.EMAIL_SMTP_PORT = 587;
EnvironmentDefault.EMAIL_SMTP_USER = '';
EnvironmentDefault.EMAIL_SMTP_PASSWORD = '';
EnvironmentDefault.SUPPORT_DRIVER = SupportDriver.None;
EnvironmentDefault.SUPPORT_FRONT_CHAT_ID = '';
EnvironmentDefault.SUPPORT_FRONT_HMAC_KEY = '';
EnvironmentDefault.LOGGER_DRIVER = LoggerDriverType.Console;
EnvironmentDefault.EXCEPTION_HANDLER_DRIVER = ExceptionHandlerDriver.Console;
EnvironmentDefault.LOG_LEVELS = ['log', 'error', 'warn'];
EnvironmentDefault.SENTRY_DSN = '';
EnvironmentDefault.DEMO_WORKSPACE_IDS = [];
EnvironmentDefault.OPENROUTER_API_KEY = '';
EnvironmentDefault.PASSWORD_RESET_TOKEN_EXPIRES_IN = '5m';
EnvironmentDefault.WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION = 30;
EnvironmentDefault.WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION = 60;
EnvironmentDefault.IS_SIGN_UP_DISABLED = false;
EnvironmentDefault.API_RATE_LIMITING_TTL = 100;
EnvironmentDefault.API_RATE_LIMITING_LIMIT = 500;
EnvironmentDefault.MUTATION_MAXIMUM_RECORD_AFFECTED = 100;
EnvironmentDefault.CACHE_STORAGE_TYPE = 'memory';
EnvironmentDefault.CACHE_STORAGE_TTL = 3600 * 24 * 7;
export { EnvironmentDefault };

View File

@ -0,0 +1,8 @@
import { ConfigurableModuleBuilder } from '@nestjs/common';
export const { ConfigurableModuleClass, MODULE_OPTIONS_TOKEN } =
new ConfigurableModuleBuilder({
moduleName: 'Environment',
})
.setClassMethodName('forRoot')
.build();

View File

@ -0,0 +1,20 @@
import { Global, Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { EnvironmentService } from './environment.service';
import { ConfigurableModuleClass } from './environment.module-definition';
import { validate } from './environment-variables';
@Global()
@Module({
imports: [
ConfigModule.forRoot({
isGlobal: true,
expandVariables: true,
validate,
}),
],
providers: [EnvironmentService],
exports: [EnvironmentService],
})
export class EnvironmentModule extends ConfigurableModuleClass {}

View File

@ -0,0 +1,26 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ConfigService } from '@nestjs/config';
import { EnvironmentService } from './environment.service';
describe('EnvironmentService', () => {
let service: EnvironmentService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
EnvironmentService,
{
provide: ConfigService,
useValue: {},
},
],
}).compile();
service = module.get<EnvironmentService>(EnvironmentService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -0,0 +1,48 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Request } from 'express';
import { EnvironmentVariables } from 'src/engine/integrations/environment/environment-variables';
import { EnvironmentDefault } from 'src/engine/integrations/environment/environment.default';
@Injectable()
export class EnvironmentService {
constructor(private configService: ConfigService) {}
get<T extends keyof EnvironmentVariables>(key: T): EnvironmentVariables[T] {
return (
this.configService.get<EnvironmentVariables[T]>(key) ??
EnvironmentDefault[key]
);
}
getServerUrl(): string {
const url = this.configService.get<string>('SERVER_URL')!;
if (url?.endsWith('/')) {
return url.substring(0, url.length - 1);
}
return url;
}
getBaseUrl(request: Request): string {
return (
this.getServerUrl() || `${request.protocol}://${request.get('host')}`
);
}
getFrontAuthCallbackUrl(): string {
return (
this.configService.get<string>('FRONT_AUTH_CALLBACK_URL') ??
this.get('FRONT_BASE_URL') + '/verify'
);
}
// TODO: check because it isn't called
getLoggerIsBufferEnabled(): boolean | undefined {
return this.configService.get<boolean>('LOGGER_IS_BUFFER_ENABLED') ?? true;
}
}

View File

@ -0,0 +1 @@
export type AwsRegion = `${string}-${string}-${number}`;

View File

@ -0,0 +1,4 @@
export enum SupportDriver {
None = 'none',
Front = 'front',
}

View File

@ -0,0 +1,12 @@
import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects/base.object-metadata';
export type CreatedObjectMetadata = {
nameSingular: string;
isCustom: boolean;
};
export class ObjectRecordCreateEvent<T extends BaseObjectMetadata> {
workspaceId: string;
createdRecord: T;
createdObjectMetadata: CreatedObjectMetadata;
}

View File

@ -0,0 +1,6 @@
import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects/base.object-metadata';
export declare class ObjectRecordDeleteEvent<T extends BaseObjectMetadata> {
workspaceId: string;
deletedRecord: T;
}

View File

@ -0,0 +1,7 @@
import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects/base.object-metadata';
export class ObjectRecordUpdateEvent<T extends BaseObjectMetadata> {
workspaceId: string;
previousRecord: T;
updatedRecord: T;
}

View File

@ -0,0 +1,12 @@
import deepEqual from 'deep-equal';
export const objectRecordChangedProperties = (
oldRecord: Record<string, any>,
newRecord: Record<string, any>,
) => {
const changedProperties = Object.keys(newRecord).filter(
(key) => !deepEqual(oldRecord[key], newRecord[key]),
);
return changedProperties;
};

View File

@ -0,0 +1,26 @@
import { Test, TestingModule } from '@nestjs/testing';
import { EXCEPTION_HANDLER_DRIVER } from 'src/engine/integrations/exception-handler/exception-handler.constants';
import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service';
describe('ExceptionHandlerService', () => {
let service: ExceptionHandlerService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
ExceptionHandlerService,
{
provide: EXCEPTION_HANDLER_DRIVER,
useValue: {},
},
],
}).compile();
service = module.get<ExceptionHandlerService>(ExceptionHandlerService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -0,0 +1,27 @@
import { ExceptionHandlerUser } from 'src/engine/integrations/exception-handler/interfaces/exception-handler-user.interface';
import { ExceptionHandlerOptions } from 'src/engine/integrations/exception-handler/interfaces/exception-handler-options.interface';
import { ExceptionHandlerDriverInterface } from 'src/engine/integrations/exception-handler/interfaces';
export class ExceptionHandlerConsoleDriver
implements ExceptionHandlerDriverInterface
{
captureExceptions(
exceptions: ReadonlyArray<any>,
options?: ExceptionHandlerOptions,
) {
console.group('Exception Captured');
console.info(options);
console.error(exceptions);
console.groupEnd();
return [];
}
captureMessage(message: string, user?: ExceptionHandlerUser): void {
console.group('Message Captured');
console.info(user);
console.info(message);
console.groupEnd();
}
}

View File

@ -0,0 +1,110 @@
import * as Sentry from '@sentry/node';
import { ProfilingIntegration } from '@sentry/profiling-node';
import { ExceptionHandlerUser } from 'src/engine/integrations/exception-handler/interfaces/exception-handler-user.interface';
import { ExceptionHandlerOptions } from 'src/engine/integrations/exception-handler/interfaces/exception-handler-options.interface';
import {
ExceptionHandlerDriverInterface,
ExceptionHandlerSentryDriverFactoryOptions,
} from 'src/engine/integrations/exception-handler/interfaces';
export class ExceptionHandlerSentryDriver
implements ExceptionHandlerDriverInterface
{
constructor(options: ExceptionHandlerSentryDriverFactoryOptions['options']) {
Sentry.init({
dsn: options.dsn,
integrations: [
new Sentry.Integrations.Http({ tracing: true }),
new Sentry.Integrations.Express({ app: options.serverInstance }),
new Sentry.Integrations.GraphQL(),
new Sentry.Integrations.Postgres(),
new ProfilingIntegration(),
],
tracesSampleRate: 0.1,
profilesSampleRate: 0.3,
environment: options.debug ? 'development' : 'production',
debug: options.debug,
});
}
captureExceptions(
exceptions: ReadonlyArray<any>,
options?: ExceptionHandlerOptions,
) {
const eventIds: string[] = [];
Sentry.withScope((scope) => {
if (options?.operation) {
scope.setTag('operation', options.operation.name);
scope.setTag('operationName', options.operation.name);
}
if (options?.document) {
scope.setExtra('document', options.document);
}
if (options?.user) {
scope.setUser({
id: options.user.id,
email: options.user.email,
firstName: options.user.firstName,
lastName: options.user.lastName,
workspaceId: options.user.workspaceId,
workspaceDisplayName: options.user.workspaceDisplayName,
});
}
for (const exception of exceptions) {
const errorPath = (exception.path ?? [])
.map((v: string | number) => (typeof v === 'number' ? '$index' : v))
.join(' > ');
if (errorPath) {
scope.addBreadcrumb({
category: 'execution-path',
message: errorPath,
level: 'debug',
});
}
const eventId = Sentry.captureException(exception, {
fingerprint: [
'graphql',
errorPath,
options?.operation?.name,
options?.operation?.type,
],
contexts: {
GraphQL: {
operationName: options?.operation?.name,
operationType: options?.operation?.type,
},
},
});
eventIds.push(eventId);
}
});
return eventIds;
}
captureMessage(message: string, user?: ExceptionHandlerUser) {
Sentry.captureMessage(message, (scope) => {
if (user) {
scope.setUser({
id: user.id,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
workspaceId: user.workspaceId,
workspaceDisplayName: user.workspaceDisplayName,
});
}
return scope;
});
}
}

View File

@ -0,0 +1 @@
export const EXCEPTION_HANDLER_DRIVER = Symbol('EXCEPTION_HANDLER_DRIVER');

View File

@ -0,0 +1,25 @@
import {
ConfigurableModuleBuilder,
FactoryProvider,
ModuleMetadata,
} from '@nestjs/common';
import { ExceptionHandlerModuleOptions } from './interfaces';
export const {
ConfigurableModuleClass,
MODULE_OPTIONS_TOKEN,
OPTIONS_TYPE,
ASYNC_OPTIONS_TYPE,
} = new ConfigurableModuleBuilder<ExceptionHandlerModuleOptions>({
moduleName: 'ExceptionHandlerModule',
})
.setClassMethodName('forRoot')
.build();
export type ExceptionHandlerModuleAsyncOptions = {
useFactory: (
...args: any[]
) => ExceptionHandlerModuleOptions | Promise<ExceptionHandlerModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;

View File

@ -0,0 +1,39 @@
import { HttpAdapterHost } from '@nestjs/core';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import { OPTIONS_TYPE } from 'src/engine/integrations/exception-handler/exception-handler.module-definition';
import { ExceptionHandlerDriver } from 'src/engine/integrations/exception-handler/interfaces';
/**
* ExceptionHandler Module factory
* @param environment
* @returns ExceptionHandlerModuleOptions
*/
export const exceptionHandlerModuleFactory = async (
environmentService: EnvironmentService,
adapterHost: HttpAdapterHost,
): Promise<typeof OPTIONS_TYPE> => {
const driverType = environmentService.get('EXCEPTION_HANDLER_DRIVER');
switch (driverType) {
case ExceptionHandlerDriver.Console: {
return {
type: ExceptionHandlerDriver.Console,
};
}
case ExceptionHandlerDriver.Sentry: {
return {
type: ExceptionHandlerDriver.Sentry,
options: {
dsn: environmentService.get('SENTRY_DSN') ?? '',
serverInstance: adapterHost.httpAdapter?.getInstance(),
debug: environmentService.get('DEBUG_MODE'),
},
};
}
default:
throw new Error(
`Invalid exception capturer driver type (${driverType}), check your .env file`,
);
}
};

View File

@ -0,0 +1,60 @@
import { DynamicModule, Global, Module } from '@nestjs/common';
import { ExceptionHandlerSentryDriver } from 'src/engine/integrations/exception-handler/drivers/sentry.driver';
import { ExceptionHandlerConsoleDriver } from 'src/engine/integrations/exception-handler/drivers/console.driver';
import { ExceptionHandlerService } from './exception-handler.service';
import { ExceptionHandlerDriver } from './interfaces';
import { EXCEPTION_HANDLER_DRIVER } from './exception-handler.constants';
import {
ConfigurableModuleClass,
OPTIONS_TYPE,
ASYNC_OPTIONS_TYPE,
} from './exception-handler.module-definition';
@Global()
@Module({
providers: [ExceptionHandlerService],
exports: [ExceptionHandlerService],
})
export class ExceptionHandlerModule extends ConfigurableModuleClass {
static forRoot(options: typeof OPTIONS_TYPE): DynamicModule {
const provider = {
provide: EXCEPTION_HANDLER_DRIVER,
useValue:
options.type === ExceptionHandlerDriver.Console
? new ExceptionHandlerConsoleDriver()
: new ExceptionHandlerSentryDriver(options.options),
};
const dynamicModule = super.forRoot(options);
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), provider],
};
}
static forRootAsync(options: typeof ASYNC_OPTIONS_TYPE): DynamicModule {
const provider = {
provide: EXCEPTION_HANDLER_DRIVER,
useFactory: async (...args: any[]) => {
const config = await options?.useFactory?.(...args);
if (!config) {
return null;
}
return config.type === ExceptionHandlerDriver.Console
? new ExceptionHandlerConsoleDriver()
: new ExceptionHandlerSentryDriver(config.options);
},
inject: options.inject || [],
};
const dynamicModule = super.forRootAsync(options);
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), provider],
};
}
}

View File

@ -0,0 +1,21 @@
import { Inject, Injectable } from '@nestjs/common';
import { ExceptionHandlerOptions } from 'src/engine/integrations/exception-handler/interfaces/exception-handler-options.interface';
import { ExceptionHandlerDriverInterface } from 'src/engine/integrations/exception-handler/interfaces';
import { EXCEPTION_HANDLER_DRIVER } from 'src/engine/integrations/exception-handler/exception-handler.constants';
@Injectable()
export class ExceptionHandlerService {
constructor(
@Inject(EXCEPTION_HANDLER_DRIVER)
private driver: ExceptionHandlerDriverInterface,
) {}
captureExceptions(
exceptions: ReadonlyArray<any>,
options?: ExceptionHandlerOptions,
): string[] {
return this.driver.captureExceptions(exceptions, options);
}
}

View File

@ -0,0 +1,129 @@
import { GraphQLError, Kind, OperationDefinitionNode, print } from 'graphql';
import {
getDocumentString,
handleStreamOrSingleExecutionResult,
OnExecuteDoneHookResultOnNextHook,
Plugin,
} from '@envelop/core';
import { GraphQLContext } from 'src/engine-graphql-config/interfaces/graphql-context.interface';
import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service';
import {
convertExceptionToGraphQLError,
filterException,
} from 'src/engine/filters/utils/global-exception-handler.util';
export type ExceptionHandlerPluginOptions = {
/**
* The exception handler service to use.
*/
exceptionHandlerService: ExceptionHandlerService;
/**
* The key of the event id in the error's extension. `null` to disable.
* @default exceptionEventId
*/
eventIdKey?: string | null;
};
export const useExceptionHandler = <PluginContext extends GraphQLContext>(
options: ExceptionHandlerPluginOptions,
): Plugin<PluginContext> => {
const eventIdKey = options.eventIdKey === null ? null : 'exceptionEventId';
function addEventId(
err: GraphQLError,
eventId: string | undefined | null,
): GraphQLError {
if (eventIdKey !== null && eventId) {
err.extensions[eventIdKey] = eventId;
}
return err;
}
return {
async onExecute({ args }) {
const exceptionHandlerService = options.exceptionHandlerService;
const rootOperation = args.document.definitions.find(
(o) => o.kind === Kind.OPERATION_DEFINITION,
) as OperationDefinitionNode;
const operationType = rootOperation.operation;
const user = args.contextValue.user;
const document = getDocumentString(args.document, print);
const opName =
args.operationName ||
rootOperation.name?.value ||
'Anonymous Operation';
return {
onExecuteDone(payload) {
const handleResult: OnExecuteDoneHookResultOnNextHook<object> = ({
result,
setResult,
}) => {
if (result.errors && result.errors.length > 0) {
const exceptions = result.errors.reduce<{
filtered: any[];
unfiltered: any[];
}>(
(acc, err) => {
// Filter out exceptions that we don't want to be captured by exception handler
if (filterException(err?.originalError ?? err)) {
acc.filtered.push(err);
} else {
acc.unfiltered.push(err);
}
return acc;
},
{
filtered: [],
unfiltered: [],
},
);
if (exceptions.unfiltered.length > 0) {
const eventIds = exceptionHandlerService.captureExceptions(
exceptions.unfiltered,
{
operation: {
name: opName,
type: operationType,
},
document,
user,
},
);
exceptions.unfiltered.map((err, i) =>
addEventId(err, eventIds?.[i]),
);
}
const concatenatedErrors = [
...exceptions.filtered,
...exceptions.unfiltered,
];
const errors = concatenatedErrors.map((err) => {
// Properly convert errors to GraphQLErrors
const graphQLError = convertExceptionToGraphQLError(
err.originalError,
);
return graphQLError;
});
setResult({
...result,
errors,
});
}
};
return handleStreamOrSingleExecutionResult(payload, handleResult);
},
};
},
};
};

View File

@ -0,0 +1,57 @@
import * as Sentry from '@sentry/node';
import {
handleStreamOrSingleExecutionResult,
Plugin,
getDocumentString,
} from '@envelop/core';
import { OperationDefinitionNode, Kind, print } from 'graphql';
import { GraphQLContext } from 'src/engine-graphql-config/graphql-config.service';
export const useSentryTracing = <
PluginContext extends GraphQLContext,
>(): Plugin<PluginContext> => {
return {
onExecute({ args }) {
const transactionName = args.operationName || 'Anonymous Operation';
const rootOperation = args.document.definitions.find(
(o) => o.kind === Kind.OPERATION_DEFINITION,
) as OperationDefinitionNode;
const operationType = rootOperation.operation;
const user = args.contextValue.user;
const workspace = args.contextValue.workspace;
const document = getDocumentString(args.document, print);
Sentry.setTags({
operationName: transactionName,
operation: operationType,
});
const scope = Sentry.getCurrentScope();
scope.setTransactionName(transactionName);
if (user) {
scope.setUser({
id: user.id,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
workspaceId: workspace?.id,
workspaceDisplayName: workspace?.displayName,
});
}
if (document) {
scope.setExtra('document', document);
}
return {
onExecuteDone(payload) {
return handleStreamOrSingleExecutionResult(payload, () => {});
},
};
},
};
};

View File

@ -0,0 +1,10 @@
import { ExceptionHandlerOptions } from './exception-handler-options.interface';
import { ExceptionHandlerUser } from './exception-handler-user.interface';
export interface ExceptionHandlerDriverInterface {
captureExceptions(
exceptions: ReadonlyArray<any>,
options?: ExceptionHandlerOptions,
): string[];
captureMessage(message: string, user?: ExceptionHandlerUser): void;
}

View File

@ -0,0 +1,12 @@
import { OperationTypeNode } from 'graphql';
import { ExceptionHandlerUser } from './exception-handler-user.interface';
export interface ExceptionHandlerOptions {
operation?: {
type: OperationTypeNode;
name: string;
};
document?: string;
user?: ExceptionHandlerUser;
}

View File

@ -0,0 +1,8 @@
export interface ExceptionHandlerUser {
id?: string;
email?: string;
firstName?: string;
lastName?: string;
workspaceId?: string;
workspaceDisplayName?: string;
}

View File

@ -0,0 +1,23 @@
import { Router } from 'express';
export enum ExceptionHandlerDriver {
Sentry = 'sentry',
Console = 'console',
}
export interface ExceptionHandlerSentryDriverFactoryOptions {
type: ExceptionHandlerDriver.Sentry;
options: {
dsn: string;
serverInstance?: Router;
debug?: boolean;
};
}
export interface ExceptionHandlerDriverFactoryOptions {
type: ExceptionHandlerDriver.Console;
}
export type ExceptionHandlerModuleOptions =
| ExceptionHandlerSentryDriverFactoryOptions
| ExceptionHandlerDriverFactoryOptions;

View File

@ -0,0 +1,2 @@
export * from './exception-handler.interface';
export * from './exception-handler-driver.interface';

View File

@ -0,0 +1,26 @@
import { Test, TestingModule } from '@nestjs/testing';
import { STORAGE_DRIVER } from 'src/engine/integrations/file-storage/file-storage.constants';
import { FileStorageService } from 'src/engine/integrations/file-storage/file-storage.service';
describe('FileStorageService', () => {
let service: FileStorageService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
FileStorageService,
{
provide: STORAGE_DRIVER,
useValue: {},
},
],
}).compile();
service = module.get<FileStorageService>(FileStorageService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -0,0 +1,11 @@
import { Readable } from 'stream';
export interface StorageDriver {
read(params: { folderPath: string; filename: string }): Promise<Readable>;
write(params: {
file: Buffer | Uint8Array | string;
name: string;
folder: string;
mimeType: string | undefined;
}): Promise<void>;
}

View File

@ -0,0 +1,57 @@
import * as fs from 'fs/promises';
import { createReadStream, existsSync } from 'fs';
import { join, dirname } from 'path';
import { Readable } from 'stream';
import { StorageDriver } from './interfaces/storage-driver.interface';
export interface LocalDriverOptions {
storagePath: string;
}
export class LocalDriver implements StorageDriver {
private options: LocalDriverOptions;
constructor(options: LocalDriverOptions) {
this.options = options;
}
async createFolder(path: string) {
if (existsSync(path)) {
return;
}
return fs.mkdir(path, { recursive: true });
}
async write(params: {
file: Buffer | Uint8Array | string;
name: string;
folder: string;
mimeType: string | undefined;
}): Promise<void> {
const filePath = join(
`${this.options.storagePath}/`,
params.folder,
params.name,
);
const folderPath = dirname(filePath);
await this.createFolder(folderPath);
await fs.writeFile(filePath, params.file);
}
async read(params: {
folderPath: string;
filename: string;
}): Promise<Readable> {
const filePath = join(
`${this.options.storagePath}/`,
params.folderPath,
params.filename,
);
return createReadStream(filePath);
}
}

View File

@ -0,0 +1,98 @@
import { Readable } from 'stream';
import {
CreateBucketCommandInput,
GetObjectCommand,
HeadBucketCommandInput,
NotFound,
PutObjectCommand,
S3,
S3ClientConfig,
} from '@aws-sdk/client-s3';
import { StorageDriver } from './interfaces/storage-driver.interface';
export interface S3DriverOptions extends S3ClientConfig {
bucketName: string;
endpoint?: string;
region: string;
}
export class S3Driver implements StorageDriver {
private s3Client: S3;
private bucketName: string;
constructor(options: S3DriverOptions) {
const { bucketName, region, endpoint, ...s3Options } = options;
if (!bucketName || !region) {
return;
}
this.s3Client = new S3({ ...s3Options, region, endpoint });
this.bucketName = bucketName;
}
public get client(): S3 {
return this.s3Client;
}
async write(params: {
file: Buffer | Uint8Array | string;
name: string;
folder: string;
mimeType: string | undefined;
}): Promise<void> {
const command = new PutObjectCommand({
Key: `${params.folder}/${params.name}`,
Body: params.file,
ContentType: params.mimeType,
Bucket: this.bucketName,
});
await this.s3Client.send(command);
}
async read(params: {
folderPath: string;
filename: string;
}): Promise<Readable> {
const command = new GetObjectCommand({
Key: `${params.folderPath}/${params.filename}`,
Bucket: this.bucketName,
});
const file = await this.s3Client.send(command);
if (!file || !file.Body || !(file.Body instanceof Readable)) {
throw new Error('Unable to get file stream');
}
return Readable.from(file.Body);
}
async checkBucketExists(args: HeadBucketCommandInput) {
try {
await this.s3Client.headBucket(args);
return true;
} catch (error) {
if (error instanceof NotFound) {
return false;
}
throw error;
}
}
async createBucket(args: CreateBucketCommandInput) {
const exist = await this.checkBucketExists({
Bucket: args.Bucket,
});
if (exist) {
return;
}
return this.s3Client.createBucket(args);
}
}

View File

@ -0,0 +1 @@
export const STORAGE_DRIVER = Symbol('STORAGE_DRIVER');

View File

@ -0,0 +1,14 @@
import { ConfigurableModuleBuilder } from '@nestjs/common';
import { FileStorageModuleOptions } from './interfaces';
export const {
ConfigurableModuleClass,
MODULE_OPTIONS_TOKEN,
OPTIONS_TYPE,
ASYNC_OPTIONS_TYPE,
} = new ConfigurableModuleBuilder<FileStorageModuleOptions>({
moduleName: 'FileStorage',
})
.setClassMethodName('forRoot')
.build();

View File

@ -0,0 +1,53 @@
import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import {
FileStorageModuleOptions,
StorageDriverType,
} from 'src/engine/integrations/file-storage/interfaces';
/**
* FileStorage Module factory
* @param environment
* @returns FileStorageModuleOptions
*/
export const fileStorageModuleFactory = async (
environmentService: EnvironmentService,
): Promise<FileStorageModuleOptions> => {
const driverType = environmentService.get('STORAGE_TYPE');
switch (driverType) {
case StorageDriverType.Local: {
const storagePath = environmentService.get('STORAGE_LOCAL_PATH');
return {
type: StorageDriverType.Local,
options: {
storagePath: process.cwd() + '/' + storagePath,
},
};
}
case StorageDriverType.S3: {
const bucketName = environmentService.get('STORAGE_S3_NAME');
const endpoint = environmentService.get('STORAGE_S3_ENDPOINT');
const region = environmentService.get('STORAGE_S3_REGION');
return {
type: StorageDriverType.S3,
options: {
bucketName: bucketName ?? '',
endpoint: endpoint,
credentials: fromNodeProviderChain({
clientConfig: { region },
}),
forcePathStyle: true,
region: region ?? '',
},
};
}
default:
throw new Error(
`Invalid storage driver type (${driverType}), check your .env file`,
);
}
};

View File

@ -0,0 +1,51 @@
import { DynamicModule, Global } from '@nestjs/common';
import { FileStorageService } from './file-storage.service';
import {
FileStorageModuleAsyncOptions,
FileStorageModuleOptions,
} from './interfaces';
import { STORAGE_DRIVER } from './file-storage.constants';
import { LocalDriver } from './drivers/local.driver';
import { S3Driver } from './drivers/s3.driver';
@Global()
export class FileStorageModule {
static forRoot(options: FileStorageModuleOptions): DynamicModule {
const provider = {
provide: STORAGE_DRIVER,
useValue:
options.type === 's3'
? new S3Driver(options.options)
: new LocalDriver(options.options),
};
return {
module: FileStorageModule,
providers: [FileStorageService, provider],
exports: [FileStorageService],
};
}
static forRootAsync(options: FileStorageModuleAsyncOptions): DynamicModule {
const provider = {
provide: STORAGE_DRIVER,
useFactory: async (...args: any[]) => {
const config = await options.useFactory(...args);
return config?.type === 's3'
? new S3Driver(config.options)
: new LocalDriver(config.options);
},
inject: options.inject || [],
};
return {
module: FileStorageModule,
imports: options.imports || [],
providers: [FileStorageService, provider],
exports: [FileStorageService],
};
}
}

View File

@ -0,0 +1,25 @@
import { Inject, Injectable } from '@nestjs/common';
import { Readable } from 'stream';
import { STORAGE_DRIVER } from './file-storage.constants';
import { StorageDriver } from './drivers/interfaces/storage-driver.interface';
@Injectable()
export class FileStorageService implements StorageDriver {
constructor(@Inject(STORAGE_DRIVER) private driver: StorageDriver) {}
write(params: {
file: string | Buffer | Uint8Array;
name: string;
folder: string;
mimeType: string | undefined;
}): Promise<void> {
return this.driver.write(params);
}
read(params: { folderPath: string; filename: string }): Promise<Readable> {
return this.driver.read(params);
}
}

View File

@ -0,0 +1,30 @@
import { FactoryProvider, ModuleMetadata } from '@nestjs/common';
import { S3DriverOptions } from 'src/engine/integrations/file-storage/drivers/s3.driver';
import { LocalDriverOptions } from 'src/engine/integrations/file-storage/drivers/local.driver';
export enum StorageDriverType {
S3 = 's3',
Local = 'local',
}
export interface S3DriverFactoryOptions {
type: StorageDriverType.S3;
options: S3DriverOptions;
}
export interface LocalDriverFactoryOptions {
type: StorageDriverType.Local;
options: LocalDriverOptions;
}
export type FileStorageModuleOptions =
| S3DriverFactoryOptions
| LocalDriverFactoryOptions;
export type FileStorageModuleAsyncOptions = {
useFactory: (
...args: any[]
) => FileStorageModuleOptions | Promise<FileStorageModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;

View File

@ -0,0 +1 @@
export * from './file-storage.interface';

View File

@ -0,0 +1,51 @@
import { Module } from '@nestjs/common';
import { HttpAdapterHost } from '@nestjs/core';
import { EventEmitterModule } from '@nestjs/event-emitter';
import { ExceptionHandlerModule } from 'src/engine/integrations/exception-handler/exception-handler.module';
import { exceptionHandlerModuleFactory } from 'src/engine/integrations/exception-handler/exception-handler.module-factory';
import { fileStorageModuleFactory } from 'src/engine/integrations/file-storage/file-storage.module-factory';
import { loggerModuleFactory } from 'src/engine/integrations/logger/logger.module-factory';
import { messageQueueModuleFactory } from 'src/engine/integrations/message-queue/message-queue.module-factory';
import { EmailModule } from 'src/engine/integrations/email/email.module';
import { emailModuleFactory } from 'src/engine/integrations/email/email.module-factory';
import { CacheStorageModule } from 'src/engine/integrations/cache-storage/cache-storage.module';
import { EnvironmentModule } from './environment/environment.module';
import { EnvironmentService } from './environment/environment.service';
import { FileStorageModule } from './file-storage/file-storage.module';
import { LoggerModule } from './logger/logger.module';
import { MessageQueueModule } from './message-queue/message-queue.module';
@Module({
imports: [
EnvironmentModule.forRoot({}),
FileStorageModule.forRootAsync({
useFactory: fileStorageModuleFactory,
inject: [EnvironmentService],
}),
LoggerModule.forRootAsync({
useFactory: loggerModuleFactory,
inject: [EnvironmentService],
}),
MessageQueueModule.forRoot({
useFactory: messageQueueModuleFactory,
inject: [EnvironmentService],
}),
ExceptionHandlerModule.forRootAsync({
useFactory: exceptionHandlerModuleFactory,
inject: [EnvironmentService, HttpAdapterHost],
}),
EmailModule.forRoot({
useFactory: emailModuleFactory,
inject: [EnvironmentService],
}),
EventEmitterModule.forRoot({
wildcard: true,
}),
CacheStorageModule,
],
exports: [],
providers: [],
})
export class IntegrationsModule {}

View File

@ -0,0 +1,26 @@
import { Test, TestingModule } from '@nestjs/testing';
import { LOGGER_DRIVER } from 'src/engine/integrations/logger/logger.constants';
import { LoggerService } from 'src/engine/integrations/logger/logger.service';
describe('LoggerService', () => {
let service: LoggerService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
LoggerService,
{
provide: LOGGER_DRIVER,
useValue: {},
},
],
}).compile();
service = module.get<LoggerService>(LoggerService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -0,0 +1 @@
export * from './logger.interface';

View File

@ -0,0 +1,12 @@
import { LogLevel } from '@nestjs/common';
export enum LoggerDriverType {
Console = 'console',
}
export interface ConsoleDriverFactoryOptions {
type: LoggerDriverType.Console;
logLevels?: LogLevel[];
}
export type LoggerModuleOptions = ConsoleDriverFactoryOptions;

View File

@ -0,0 +1 @@
export const LOGGER_DRIVER = Symbol('LOGGER_DRIVER');

View File

@ -0,0 +1,25 @@
import {
ConfigurableModuleBuilder,
FactoryProvider,
ModuleMetadata,
} from '@nestjs/common';
import { LoggerModuleOptions } from './interfaces';
export const {
ConfigurableModuleClass,
MODULE_OPTIONS_TOKEN,
OPTIONS_TYPE,
ASYNC_OPTIONS_TYPE,
} = new ConfigurableModuleBuilder<LoggerModuleOptions>({
moduleName: 'LoggerService',
})
.setClassMethodName('forRoot')
.build();
export type LoggerModuleAsyncOptions = {
useFactory: (
...args: any[]
) => LoggerModuleOptions | Promise<LoggerModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;

View File

@ -0,0 +1,30 @@
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import {
LoggerModuleOptions,
LoggerDriverType,
} from 'src/engine/integrations/logger/interfaces';
/**
* Logger Module factory
* @param environment
* @returns LoggerModuleOptions
*/
export const loggerModuleFactory = async (
environmentService: EnvironmentService,
): Promise<LoggerModuleOptions> => {
const driverType = environmentService.get('LOGGER_DRIVER');
const logLevels = environmentService.get('LOG_LEVELS');
switch (driverType) {
case LoggerDriverType.Console: {
return {
type: LoggerDriverType.Console,
logLevels: logLevels,
};
}
default:
throw new Error(
`Invalid logger driver type (${driverType}), check your .env file`,
);
}
};

View File

@ -0,0 +1,65 @@
import { DynamicModule, Global, ConsoleLogger, Module } from '@nestjs/common';
import { LoggerDriverType } from 'src/engine/integrations/logger/interfaces';
import { LoggerService } from './logger.service';
import { LOGGER_DRIVER } from './logger.constants';
import {
ASYNC_OPTIONS_TYPE,
ConfigurableModuleClass,
OPTIONS_TYPE,
} from './logger.module-definition';
@Global()
@Module({
providers: [LoggerService],
exports: [LoggerService],
})
export class LoggerModule extends ConfigurableModuleClass {
static forRoot(options: typeof OPTIONS_TYPE): DynamicModule {
const provider = {
provide: LOGGER_DRIVER,
useValue:
options.type === LoggerDriverType.Console
? new ConsoleLogger()
: undefined,
};
const dynamicModule = super.forRoot(options);
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), provider],
};
}
static forRootAsync(options: typeof ASYNC_OPTIONS_TYPE): DynamicModule {
const provider = {
provide: LOGGER_DRIVER,
useFactory: async (...args: any[]) => {
const config = await options?.useFactory?.(...args);
if (!config) {
return null;
}
const logLevels = config.logLevels ?? [];
const logger =
config?.type === LoggerDriverType.Console
? new ConsoleLogger()
: undefined;
logger?.setLogLevels(logLevels);
return logger;
},
inject: options.inject || [],
};
const dynamicModule = super.forRootAsync(options);
return {
...dynamicModule,
providers: [...(dynamicModule.providers ?? []), provider],
};
}
}

View File

@ -0,0 +1,49 @@
import {
Inject,
Injectable,
LogLevel,
LoggerService as LoggerServiceInterface,
} from '@nestjs/common';
import { LOGGER_DRIVER } from './logger.constants';
@Injectable()
export class LoggerService implements LoggerServiceInterface {
constructor(@Inject(LOGGER_DRIVER) private driver: LoggerServiceInterface) {}
log(message: any, category: string, ...optionalParams: any[]) {
this.driver.log.apply(this.driver, [message, category, ...optionalParams]);
}
error(message: any, category: string, ...optionalParams: any[]) {
this.driver.error.apply(this.driver, [
message,
category,
...optionalParams,
]);
}
warn(message: any, category: string, ...optionalParams: any[]) {
this.driver.warn.apply(this.driver, [message, category, ...optionalParams]);
}
debug?(message: any, category: string, ...optionalParams: any[]) {
this.driver.debug?.apply(this.driver, [
message,
category,
...optionalParams,
]);
}
verbose?(message: any, category: string, ...optionalParams: any[]) {
this.driver.verbose?.apply(this.driver, [
message,
category,
...optionalParams,
]);
}
setLogLevels(levels: LogLevel[]) {
this.driver.setLogLevels?.apply(this.driver, [levels]);
}
}

View File

@ -0,0 +1,104 @@
import { Queue, QueueOptions, Worker } from 'bullmq';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueDriver } from './interfaces/message-queue-driver.interface';
export type BullMQDriverOptions = QueueOptions;
export class BullMQDriver implements MessageQueueDriver {
private queueMap: Record<MessageQueue, Queue> = {} as Record<
MessageQueue,
Queue
>;
private workerMap: Record<MessageQueue, Worker> = {} as Record<
MessageQueue,
Worker
>;
constructor(private options: BullMQDriverOptions) {}
register(queueName: MessageQueue): void {
this.queueMap[queueName] = new Queue(queueName, this.options);
}
async stop() {
const workers = Object.values(this.workerMap);
const queues = Object.values(this.queueMap);
await Promise.all([
...queues.map((q) => q.close()),
...workers.map((w) => w.close()),
]);
}
async work<T>(
queueName: MessageQueue,
handler: ({ data, id }: { data: T; id: string }) => Promise<void>,
) {
const worker = new Worker(
queueName,
async (job) => {
await handler(job as { data: T; id: string });
},
this.options,
);
this.workerMap[queueName] = worker;
}
async addCron<T>(
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
): Promise<void> {
if (!this.queueMap[queueName]) {
throw new Error(
`Queue ${queueName} is not registered, make sure you have added it as a queue provider`,
);
}
const queueOptions = {
jobId: options?.id,
priority: options?.priority,
repeat: {
pattern,
},
};
await this.queueMap[queueName].add(jobName, data, queueOptions);
}
async removeCron(
queueName: MessageQueue,
jobName: string,
pattern: string,
): Promise<void> {
await this.queueMap[queueName].removeRepeatable(jobName, {
pattern,
});
}
async add<T>(
queueName: MessageQueue,
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void> {
if (!this.queueMap[queueName]) {
throw new Error(
`Queue ${queueName} is not registered, make sure you have added it as a queue provider`,
);
}
const queueOptions = {
jobId: options?.id,
priority: options?.priority,
attempts: 1 + (options?.retryLimit || 0),
};
await this.queueMap[queueName].add(jobName, data, queueOptions);
}
}

View File

@ -0,0 +1,5 @@
export interface QueueJobOptions {
id?: string;
priority?: number;
retryLimit?: number;
}

View File

@ -0,0 +1,27 @@
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
export interface MessageQueueDriver {
add<T extends MessageQueueJobData>(
queueName: MessageQueue,
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void>;
work<T extends MessageQueueJobData>(
queueName: MessageQueue,
handler: ({ data, id }: { data: T; id: string }) => Promise<void> | void,
);
addCron<T extends MessageQueueJobData | undefined>(
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
);
removeCron(queueName: MessageQueue, jobName: string, pattern?: string);
stop?(): Promise<void>;
register?(queueName: MessageQueue): void;
}

View File

@ -0,0 +1,74 @@
import PgBoss from 'pg-boss';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueDriver } from './interfaces/message-queue-driver.interface';
export type PgBossDriverOptions = PgBoss.ConstructorOptions;
export class PgBossDriver implements MessageQueueDriver {
private pgBoss: PgBoss;
constructor(options: PgBossDriverOptions) {
this.pgBoss = new PgBoss(options);
}
async stop() {
await this.pgBoss.stop();
}
async init(): Promise<void> {
await this.pgBoss.start();
}
async work<T>(
queueName: string,
handler: ({ data, id }: { data: T; id: string }) => Promise<void>,
) {
return this.pgBoss.work(`${queueName}.*`, handler);
}
async addCron<T>(
queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
): Promise<void> {
await this.pgBoss.schedule(
`${queueName}.${jobName}`,
pattern,
data as object,
options
? {
...options,
singletonKey: options?.id,
}
: {},
);
}
async removeCron(queueName: MessageQueue, jobName: string): Promise<void> {
await this.pgBoss.unschedule(`${queueName}.${jobName}`);
}
async add<T>(
queueName: MessageQueue,
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void> {
await this.pgBoss.send(
`${queueName}.${jobName}`,
data as object,
options
? {
...options,
singletonKey: options?.id,
}
: {},
);
}
}

View File

@ -0,0 +1,58 @@
import { ModuleRef } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
import {
MessageQueueCronJobData,
MessageQueueJob,
MessageQueueJobData,
} from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { getJobClassName } from 'src/engine/integrations/message-queue/utils/get-job-class-name.util';
export class SyncDriver implements MessageQueueDriver {
private readonly logger = new Logger(SyncDriver.name);
constructor(private readonly jobsModuleRef: ModuleRef) {}
async add<T extends MessageQueueJobData>(
_queueName: MessageQueue,
jobName: string,
data: T,
): Promise<void> {
const jobClassName = getJobClassName(jobName);
const job: MessageQueueJob<MessageQueueJobData> = this.jobsModuleRef.get(
jobClassName,
{ strict: true },
);
await job.handle(data);
}
async addCron<T extends MessageQueueJobData | undefined>(
_queueName: MessageQueue,
jobName: string,
data: T,
pattern: string,
): Promise<void> {
this.logger.log(`Running '${pattern}' cron job with SyncDriver`);
const jobClassName = getJobClassName(jobName);
const job: MessageQueueCronJobData<MessageQueueJobData | undefined> =
this.jobsModuleRef.get(jobClassName, {
strict: true,
});
await job.handle(data);
}
async removeCron(_queueName: MessageQueue, jobName: string) {
this.logger.log(`Removing '${jobName}' cron job with SyncDriver`);
return;
}
work() {
return;
}
}

View File

@ -0,0 +1 @@
export * from './message-queue.interface';

View File

@ -0,0 +1,13 @@
export interface MessageQueueJob<T extends MessageQueueJobData | undefined> {
handle(data: T): Promise<void> | void;
}
export interface MessageQueueCronJobData<
T extends MessageQueueJobData | undefined,
> {
handle(data: T): Promise<void> | void;
}
export interface MessageQueueJobData {
[key: string]: any;
}

View File

@ -0,0 +1,37 @@
import { FactoryProvider, ModuleMetadata } from '@nestjs/common';
import { BullMQDriverOptions } from 'src/engine/integrations/message-queue/drivers/bullmq.driver';
import { PgBossDriverOptions } from 'src/engine/integrations/message-queue/drivers/pg-boss.driver';
export enum MessageQueueDriverType {
PgBoss = 'pg-boss',
BullMQ = 'bull-mq',
Sync = 'sync',
}
export interface PgBossDriverFactoryOptions {
type: MessageQueueDriverType.PgBoss;
options: PgBossDriverOptions;
}
export interface BullMQDriverFactoryOptions {
type: MessageQueueDriverType.BullMQ;
options: BullMQDriverOptions;
}
export interface SyncDriverFactoryOptions {
type: MessageQueueDriverType.Sync;
options: Record<string, any>;
}
export type MessageQueueModuleOptions =
| PgBossDriverFactoryOptions
| BullMQDriverFactoryOptions
| SyncDriverFactoryOptions;
export type MessageQueueModuleAsyncOptions = {
useFactory: (
...args: any[]
) => MessageQueueModuleOptions | Promise<MessageQueueModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;

View File

@ -0,0 +1,132 @@
import { Module } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { HttpModule } from '@nestjs/axios';
import { TypeOrmModule } from '@nestjs/typeorm';
import { GmailFullSyncJob } from 'src/modules/messaging/jobs/gmail-full-sync.job';
import { CallWebhookJobsJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job';
import { CallWebhookJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { ObjectMetadataModule } from 'src/engine-metadata/object-metadata/object-metadata.module';
import { DataSourceModule } from 'src/engine-metadata/data-source/data-source.module';
import { CleanInactiveWorkspaceJob } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-inactive-workspace.job';
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
import { MessagingModule } from 'src/modules/messaging/messaging.module';
import { GmailPartialSyncJob } from 'src/modules/messaging/jobs/gmail-partial-sync.job';
import { EmailSenderJob } from 'src/engine/integrations/email/email-sender.job';
import { UserModule } from 'src/engine/modules/user/user.module';
import { EnvironmentModule } from 'src/engine/integrations/environment/environment.module';
import { FetchAllWorkspacesMessagesJob } from 'src/modules/messaging/commands/crons/fetch-all-workspaces-messages.job';
import { ConnectedAccountModule } from 'src/modules/connected-account/repositories/connected-account/connected-account.module';
import { MatchMessageParticipantJob } from 'src/modules/messaging/jobs/match-message-participant.job';
import { CreateCompaniesAndContactsAfterSyncJob } from 'src/modules/messaging/jobs/create-companies-and-contacts-after-sync.job';
import { CreateCompaniesAndContactsModule } from 'src/modules/connected-account/auto-companies-and-contacts-creation/create-company-and-contact/create-company-and-contact.module';
import { MessageChannelModule } from 'src/modules/messaging/repositories/message-channel/message-channel.module';
import { MessageParticipantModule } from 'src/modules/messaging/repositories/message-participant/message-participant.module';
import { DataSeedDemoWorkspaceModule } from 'src/database/commands/data-seed-demo-workspace/data-seed-demo-workspace.module';
import { DataSeedDemoWorkspaceJob } from 'src/database/commands/data-seed-demo-workspace/jobs/data-seed-demo-workspace.job';
import { DeleteConnectedAccountAssociatedMessagingDataJob } from 'src/modules/messaging/jobs/delete-connected-account-associated-messaging-data.job';
import { ThreadCleanerModule } from 'src/modules/messaging/services/thread-cleaner/thread-cleaner.module';
import { UpdateSubscriptionJob } from 'src/engine/modules/billing/jobs/update-subscription.job';
import { BillingModule } from 'src/engine/modules/billing/billing.module';
import { UserWorkspaceModule } from 'src/engine/modules/user-workspace/user-workspace.module';
import { StripeModule } from 'src/engine/modules/billing/stripe/stripe.module';
import { Workspace } from 'src/engine/modules/workspace/workspace.entity';
import { FeatureFlagEntity } from 'src/engine/modules/feature-flag/feature-flag.entity';
import { CalendarModule } from 'src/modules/calendar/calendar.module';
import { DataSourceEntity } from 'src/engine-metadata/data-source/data-source.entity';
import { GoogleCalendarFullSyncJob } from 'src/modules/calendar/jobs/google-calendar-full-sync.job';
import { CalendarEventCleanerModule } from 'src/modules/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
import { RecordPositionBackfillJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/record-position-backfill.job';
import { RecordPositionBackfillModule } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-module';
import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/modules/messaging/jobs/delete-connected-account-associated-calendar-data.job';
@Module({
imports: [
BillingModule,
DataSourceModule,
ConnectedAccountModule,
CreateCompaniesAndContactsModule,
DataSeedDemoWorkspaceModule,
EnvironmentModule,
HttpModule,
MessagingModule,
MessageParticipantModule,
MessageChannelModule,
CalendarModule,
ObjectMetadataModule,
StripeModule,
ThreadCleanerModule,
CalendarEventCleanerModule,
TypeORMModule,
TypeOrmModule.forFeature([Workspace, FeatureFlagEntity], 'core'),
TypeOrmModule.forFeature([DataSourceEntity], 'metadata'),
UserModule,
UserWorkspaceModule,
WorkspaceDataSourceModule,
RecordPositionBackfillModule,
],
providers: [
{
provide: GmailFullSyncJob.name,
useClass: GmailFullSyncJob,
},
{
provide: GmailPartialSyncJob.name,
useClass: GmailPartialSyncJob,
},
{
provide: GoogleCalendarFullSyncJob.name,
useClass: GoogleCalendarFullSyncJob,
},
{
provide: CallWebhookJobsJob.name,
useClass: CallWebhookJobsJob,
},
{
provide: CallWebhookJob.name,
useClass: CallWebhookJob,
},
{
provide: CleanInactiveWorkspaceJob.name,
useClass: CleanInactiveWorkspaceJob,
},
{ provide: EmailSenderJob.name, useClass: EmailSenderJob },
{
provide: FetchAllWorkspacesMessagesJob.name,
useClass: FetchAllWorkspacesMessagesJob,
},
{
provide: MatchMessageParticipantJob.name,
useClass: MatchMessageParticipantJob,
},
{
provide: CreateCompaniesAndContactsAfterSyncJob.name,
useClass: CreateCompaniesAndContactsAfterSyncJob,
},
{
provide: DataSeedDemoWorkspaceJob.name,
useClass: DataSeedDemoWorkspaceJob,
},
{
provide: DeleteConnectedAccountAssociatedMessagingDataJob.name,
useClass: DeleteConnectedAccountAssociatedMessagingDataJob,
},
{
provide: DeleteConnectedAccountAssociatedCalendarDataJob.name,
useClass: DeleteConnectedAccountAssociatedCalendarDataJob,
},
{ provide: UpdateSubscriptionJob.name, useClass: UpdateSubscriptionJob },
{
provide: RecordPositionBackfillJob.name,
useClass: RecordPositionBackfillJob,
},
],
})
export class JobsModule {
static moduleRef: ModuleRef;
constructor(private moduleRef: ModuleRef) {
JobsModule.moduleRef = this.moduleRef;
}
}

View File

@ -0,0 +1,12 @@
export const QUEUE_DRIVER = Symbol('QUEUE_DRIVER');
export enum MessageQueue {
taskAssignedQueue = 'task-assigned-queue',
messagingQueue = 'messaging-queue',
webhookQueue = 'webhook-queue',
cronQueue = 'cron-queue',
emailQueue = 'email-queue',
calendarQueue = 'calendar-queue',
billingQueue = 'billing-queue',
recordPositionBackfillQueue = 'record-position-backfill-queue',
}

View File

@ -0,0 +1,53 @@
import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
import {
MessageQueueDriverType,
MessageQueueModuleOptions,
} from 'src/engine/integrations/message-queue/interfaces';
/**
* MessageQueue Module factory
* @param environment
* @returns MessageQueueModuleOptions
*/
export const messageQueueModuleFactory = async (
environmentService: EnvironmentService,
): Promise<MessageQueueModuleOptions> => {
const driverType = environmentService.get('MESSAGE_QUEUE_TYPE');
switch (driverType) {
case MessageQueueDriverType.Sync: {
return {
type: MessageQueueDriverType.Sync,
options: {},
};
}
case MessageQueueDriverType.PgBoss: {
const connectionString = environmentService.get('PG_DATABASE_URL');
return {
type: MessageQueueDriverType.PgBoss,
options: {
connectionString,
},
};
}
case MessageQueueDriverType.BullMQ: {
const host = environmentService.get('REDIS_HOST');
const port = environmentService.get('REDIS_PORT');
return {
type: MessageQueueDriverType.BullMQ,
options: {
connection: {
host,
port,
},
},
};
}
default:
throw new Error(
`Invalid message queue driver type (${driverType}), check your .env file`,
);
}
};

View File

@ -0,0 +1,61 @@
import { DynamicModule, Global } from '@nestjs/common';
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
import {
MessageQueueDriverType,
MessageQueueModuleAsyncOptions,
} from 'src/engine/integrations/message-queue/interfaces';
import {
MessageQueue,
QUEUE_DRIVER,
} from 'src/engine/integrations/message-queue/message-queue.constants';
import { PgBossDriver } from 'src/engine/integrations/message-queue/drivers/pg-boss.driver';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { BullMQDriver } from 'src/engine/integrations/message-queue/drivers/bullmq.driver';
import { SyncDriver } from 'src/engine/integrations/message-queue/drivers/sync.driver';
import { JobsModule } from 'src/engine/integrations/message-queue/jobs.module';
@Global()
export class MessageQueueModule {
static forRoot(options: MessageQueueModuleAsyncOptions): DynamicModule {
const providers = [
...Object.values(MessageQueue).map((queue) => ({
provide: queue,
useFactory: (driver: MessageQueueDriver) => {
return new MessageQueueService(driver, queue);
},
inject: [QUEUE_DRIVER],
})),
{
provide: QUEUE_DRIVER,
useFactory: async (...args: any[]) => {
const config = await options.useFactory(...args);
switch (config.type) {
case MessageQueueDriverType.PgBoss:
const boss = new PgBossDriver(config.options);
await boss.init();
return boss;
case MessageQueueDriverType.BullMQ:
return new BullMQDriver(config.options);
default:
return new SyncDriver(JobsModule.moduleRef);
}
},
inject: options.inject || [],
},
];
return {
module: MessageQueueModule,
imports: [JobsModule, ...(options.imports || [])],
providers,
exports: Object.values(MessageQueue),
};
}
}

View File

@ -0,0 +1,46 @@
import { Test, TestingModule } from '@nestjs/testing';
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
import {
QUEUE_DRIVER,
MessageQueue,
} from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
describe('MessageQueueTaskAssigned queue', () => {
let service: MessageQueueService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
{
provide: MessageQueue.taskAssignedQueue,
useFactory: (driver: MessageQueueDriver) => {
return new MessageQueueService(
driver,
MessageQueue.taskAssignedQueue,
);
},
inject: [QUEUE_DRIVER],
},
{
provide: QUEUE_DRIVER,
useValue: {},
},
],
}).compile();
service = module.get<MessageQueueService>(MessageQueue.taskAssignedQueue);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
it('should contain the topic and driver', () => {
expect(service).toEqual({
driver: {},
queueName: MessageQueue.taskAssignedQueue,
});
});
});

View File

@ -0,0 +1,55 @@
import { Inject, Injectable, OnModuleDestroy } from '@nestjs/common';
import { QueueJobOptions } from 'src/engine/integrations/message-queue/drivers/interfaces/job-options.interface';
import { MessageQueueDriver } from 'src/engine/integrations/message-queue/drivers/interfaces/message-queue-driver.interface';
import { MessageQueueJobData } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import {
MessageQueue,
QUEUE_DRIVER,
} from 'src/engine/integrations/message-queue/message-queue.constants';
@Injectable()
export class MessageQueueService implements OnModuleDestroy {
constructor(
@Inject(QUEUE_DRIVER) protected driver: MessageQueueDriver,
protected queueName: MessageQueue,
) {
if (typeof this.driver.register === 'function') {
this.driver.register(queueName);
}
}
async onModuleDestroy() {
if (typeof this.driver.stop === 'function') {
await this.driver.stop();
}
}
add<T extends MessageQueueJobData>(
jobName: string,
data: T,
options?: QueueJobOptions,
): Promise<void> {
return this.driver.add(this.queueName, jobName, data, options);
}
addCron<T extends MessageQueueJobData | undefined>(
jobName: string,
data: T,
pattern: string,
options?: QueueJobOptions,
): Promise<void> {
return this.driver.addCron(this.queueName, jobName, data, pattern, options);
}
removeCron(jobName: string, pattern: string): Promise<void> {
return this.driver.removeCron(this.queueName, jobName, pattern);
}
work<T extends MessageQueueJobData>(
handler: ({ data, id }: { data: T; id: string }) => Promise<void> | void,
) {
return this.driver.work(this.queueName, handler);
}
}

View File

@ -0,0 +1,5 @@
export function getJobClassName(name: string): string {
const [, jobName] = name.split('.') ?? [];
return jobName ?? name;
}