diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index ebc09d0ef..37fb25af7 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -77,4 +77,4 @@ FRONTEND_URL=http://localhost:3001 # CLOUDFLARE_WEBHOOK_SECRET= # IS_CONFIG_VARIABLES_IN_DB_ENABLED=false # ANALYTICS_ENABLED= -# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty \ No newline at end of file +# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 9a508f3f1..dd13be41d 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -25,6 +25,7 @@ "@nestjs/cache-manager": "^2.2.1", "@nestjs/devtools-integration": "^0.1.6", "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch", + "@nestjs/schedule": "^3.0.0", "@node-saml/passport-saml": "^5.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.200.0", diff --git a/packages/twenty-server/src/database/typeorm/typeorm.module.ts b/packages/twenty-server/src/database/typeorm/typeorm.module.ts index 88a9b26e3..11a590405 100644 --- a/packages/twenty-server/src/database/typeorm/typeorm.module.ts +++ b/packages/twenty-server/src/database/typeorm/typeorm.module.ts @@ -20,6 +20,7 @@ const coreTypeORMFactory = async (): Promise => ({ @Module({ imports: [ + TwentyConfigModule, TypeOrmModule.forRootAsync({ useFactory: metadataTypeORMFactory, name: 'metadata', @@ -28,7 +29,6 @@ const coreTypeORMFactory = async (): Promise => ({ useFactory: coreTypeORMFactory, name: 'core', }), - TwentyConfigModule, ], providers: [TypeORMService], exports: [TypeORMService], diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index e5e2d2b87..d5ef3c18c 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { HttpAdapterHost } from '@nestjs/core'; import { EventEmitterModule } from '@nestjs/event-emitter'; +import { WorkspaceQueryRunnerModule } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.module'; import { ActorModule } from 'src/engine/core-modules/actor/actor.module'; import { AdminPanelModule } from 'src/engine/core-modules/admin-panel/admin-panel.module'; import { AppTokenModule } from 'src/engine/core-modules/app-token/app-token.module'; @@ -47,9 +48,8 @@ import { WorkflowApiModule } from 'src/engine/core-modules/workflow/workflow-api import { WorkspaceInvitationModule } from 'src/engine/core-modules/workspace-invitation/workspace-invitation.module'; import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module'; import { RoleModule } from 'src/engine/metadata-modules/role/role.module'; -import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; -import { WorkspaceQueryRunnerModule } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.module'; import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module'; +import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; import { AnalyticsModule } from './analytics/analytics.module'; import { ClientConfigModule } from './client-config/client-config.module'; @@ -57,6 +57,7 @@ import { FileModule } from './file/file.module'; @Module({ imports: [ + TwentyConfigModule.forRoot(), HealthModule, AnalyticsModule, AuthModule, @@ -81,7 +82,6 @@ import { FileModule } from './file/file.module'; AdminPanelModule, LabModule, RoleModule, - TwentyConfigModule, RedisClientModule, WorkspaceQueryRunnerModule, SubscriptionsModule, diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts index f2d109931..1f18acaf4 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts @@ -3,11 +3,10 @@ import { Module } from '@nestjs/common'; import { FileUploadResolver } from 'src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver'; import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service'; import { FileModule } from 'src/engine/core-modules/file/file.module'; -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; @Module({ imports: [FileModule], - providers: [FileUploadService, FileUploadResolver, TwentyConfigService], + providers: [FileUploadService, FileUploadResolver], exports: [FileUploadService, FileUploadResolver], }) export class FileUploadModule {} diff --git a/packages/twenty-server/src/engine/core-modules/file/file.module.ts b/packages/twenty-server/src/engine/core-modules/file/file.module.ts index b8a3b356b..e48f16c24 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file.module.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file.module.ts @@ -6,7 +6,6 @@ import { FileWorkspaceFolderDeletionJob } from 'src/engine/core-modules/file/job import { FileAttachmentListener } from 'src/engine/core-modules/file/listeners/file-attachment.listener'; import { FileWorkspaceMemberListener } from 'src/engine/core-modules/file/listeners/file-workspace-member.listener'; import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module'; -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { FileController } from './controllers/file.controller'; import { FileService } from './services/file.service'; @@ -15,7 +14,6 @@ import { FileService } from './services/file.service'; imports: [JwtModule], providers: [ FileService, - TwentyConfigService, FilePathGuard, FileAttachmentListener, FileWorkspaceMemberListener, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts new file mode 100644 index 000000000..33a219a80 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -0,0 +1,196 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +describe('ConfigCacheService', () => { + let service: ConfigCacheService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ConfigCacheService], + }).compile(); + + service = module.get(ConfigCacheService); + }); + + afterEach(() => { + service.onModuleDestroy(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('get and set', () => { + it('should set and get a value from cache', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + + service.set(key, value); + const result = service.get(key); + + expect(result).toBe(value); + }); + + it('should return undefined for non-existent key', () => { + const result = service.get( + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ); + + expect(result).toBeUndefined(); + }); + + it('should handle different value types', () => { + const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const numberKey = 'NODE_PORT' as keyof ConfigVariables; + + service.set(booleanKey, true); + service.set(stringKey, 'test@example.com'); + service.set(numberKey, 3000); + + expect(service.get(booleanKey)).toBe(true); + expect(service.get(stringKey)).toBe('test@example.com'); + expect(service.get(numberKey)).toBe(3000); + }); + }); + + describe('negative lookup cache', () => { + it('should check if a negative cache entry exists', () => { + const key = 'TEST_KEY' as keyof ConfigVariables; + + service.markKeyAsMissing(key); + const result = service.isKeyKnownMissing(key); + + expect(result).toBe(true); + }); + + it('should return false for negative cache entry check when not in cache', () => { + const key = 'NON_EXISTENT_KEY' as keyof ConfigVariables; + + const result = service.isKeyKnownMissing(key); + + expect(result).toBe(false); + }); + }); + + describe('clear operations', () => { + it('should clear specific key', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + service.clear(key1); + + expect(service.get(key1)).toBeUndefined(); + expect(service.get(key2)).toBe('test@example.com'); + }); + + it('should clear all entries', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + service.clearAll(); + + expect(service.get(key1)).toBeUndefined(); + expect(service.get(key2)).toBeUndefined(); + }); + }); + + describe('getCacheInfo', () => { + it('should return correct cache information', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const key3 = 'NODE_PORT' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + service.markKeyAsMissing(key3); + + const info = service.getCacheInfo(); + + expect(info.foundConfigValues).toBe(2); + expect(info.knownMissingKeys).toBe(1); + expect(info.cacheKeys).toContain(key1); + expect(info.cacheKeys).toContain(key2); + expect(info.cacheKeys).not.toContain(key3); + expect(service.isKeyKnownMissing(key3)).toBe(true); + }); + + it('should properly count cache entries', () => { + const key1 = 'KEY1' as keyof ConfigVariables; + const key2 = 'KEY2' as keyof ConfigVariables; + const key3 = 'KEY3' as keyof ConfigVariables; + + // Add some values to the cache + service.set(key1, 'value1'); + service.set(key2, 'value2'); + service.markKeyAsMissing(key3); + + const cacheInfo = service.getCacheInfo(); + + expect(cacheInfo.foundConfigValues).toBe(2); + expect(cacheInfo.knownMissingKeys).toBe(1); + expect(cacheInfo.cacheKeys).toContain(key1); + expect(cacheInfo.cacheKeys).toContain(key2); + expect(service.isKeyKnownMissing(key3)).toBe(true); + }); + }); + + describe('module lifecycle', () => { + it('should clear cache on module destroy', () => { + const key = 'TEST_KEY' as keyof ConfigVariables; + + service.set(key, 'test'); + + service.onModuleDestroy(); + + expect(service.get(key)).toBeUndefined(); + }); + }); + + describe('getAllKeys', () => { + it('should return all keys from both positive and negative caches', () => { + const positiveKey1 = 'POSITIVE_KEY1' as keyof ConfigVariables; + const positiveKey2 = 'POSITIVE_KEY2' as keyof ConfigVariables; + const negativeKey = 'NEGATIVE_KEY' as keyof ConfigVariables; + + // Set up keys + service.set(positiveKey1, 'value1'); + service.set(positiveKey2, 'value2'); + service.markKeyAsMissing(negativeKey); + + const allKeys = service.getAllKeys(); + + expect(allKeys).toContain(positiveKey1); + expect(allKeys).toContain(positiveKey2); + expect(allKeys).toContain(negativeKey); + }); + + it('should return empty array when no keys exist', () => { + const allKeys = service.getAllKeys(); + + expect(allKeys).toHaveLength(0); + }); + + it('should not have duplicates if a key somehow exists in both caches', () => { + const key = 'DUPLICATE_KEY' as keyof ConfigVariables; + + // First add to positive cache + service.set(key, 'value'); + + // Then force it into negative cache (normally this would remove from positive) + // We're bypassing normal behavior for testing edge cases + service.addToMissingKeysForTesting(key); + + const allKeys = service.getAllKeys(); + + // Should only appear once in the result + expect(allKeys.filter((k) => k === key)).toHaveLength(1); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts new file mode 100644 index 000000000..58fbae2b4 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -0,0 +1,86 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; + +import { + ConfigCacheEntry, + ConfigKey, + ConfigValue, +} from './interfaces/config-cache-entry.interface'; + +@Injectable() +export class ConfigCacheService implements OnModuleDestroy { + private readonly logger = new Logger(ConfigCacheService.name); + private readonly foundConfigValuesCache: Map< + ConfigKey, + ConfigCacheEntry + >; + private readonly knownMissingKeysCache: Set; + + constructor() { + this.foundConfigValuesCache = new Map(); + this.knownMissingKeysCache = new Set(); + } + + get(key: T): ConfigValue | undefined { + const entry = this.foundConfigValuesCache.get(key); + + if (!entry) { + return undefined; + } + + return entry.value as ConfigValue; + } + + isKeyKnownMissing(key: ConfigKey): boolean { + return this.knownMissingKeysCache.has(key); + } + + set(key: T, value: ConfigValue): void { + this.foundConfigValuesCache.set(key, { value }); + this.knownMissingKeysCache.delete(key); + } + + markKeyAsMissing(key: ConfigKey): void { + this.knownMissingKeysCache.add(key); + this.foundConfigValuesCache.delete(key); + } + + clear(key: ConfigKey): void { + this.foundConfigValuesCache.delete(key); + this.knownMissingKeysCache.delete(key); + } + + clearAll(): void { + this.foundConfigValuesCache.clear(); + this.knownMissingKeysCache.clear(); + } + + getCacheInfo(): { + foundConfigValues: number; + knownMissingKeys: number; + cacheKeys: string[]; + } { + return { + foundConfigValues: this.foundConfigValuesCache.size, + knownMissingKeys: this.knownMissingKeysCache.size, + cacheKeys: Array.from(this.foundConfigValuesCache.keys()), + }; + } + + onModuleDestroy() { + this.clearAll(); + } + + getAllKeys(): ConfigKey[] { + const foundKeys = Array.from(this.foundConfigValuesCache.keys()); + const missingKeys = Array.from(this.knownMissingKeysCache); + + return [...new Set([...foundKeys, ...missingKeys])]; + } + + /** + * Helper method for testing edge cases + */ + addToMissingKeysForTesting(key: ConfigKey): void { + this.knownMissingKeysCache.add(key); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts new file mode 100644 index 000000000..988e39cc7 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts @@ -0,0 +1,8 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +export type ConfigKey = keyof ConfigVariables; +export type ConfigValue = ConfigVariables[T]; + +export interface ConfigCacheEntry { + value: ConfigValue; +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index 42cf6a575..2637fa6c8 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -2,12 +2,8 @@ import { LogLevel, Logger } from '@nestjs/common'; import { plainToClass } from 'class-transformer'; import { - IsBoolean, IsDefined, - IsEnum, - IsNumber, IsOptional, - IsString, IsUrl, ValidateIf, ValidationError, @@ -28,7 +24,6 @@ import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfac import { LoggerDriverType } from 'src/engine/core-modules/logger/interfaces'; import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type'; import { ServerlessDriverType } from 'src/engine/core-modules/serverless/serverless.interface'; -import { CastToBoolean } from 'src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator'; import { CastToLogLevelArray } from 'src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator'; import { CastToMeterDriverArray } from 'src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator'; import { CastToPositiveNumber } from 'src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator'; @@ -44,35 +39,33 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Enable or disable password authentication for users', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() AUTH_PASSWORD_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Prefills tim@apple.dev in the login form, used in local development for quicker sign-in', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() @ValidateIf((env) => env.AUTH_PASSWORD_ENABLED) SIGN_IN_PREFILLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Require email verification for user accounts', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_EMAIL_VERIFICATION_REQUIRED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the email verification token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -81,6 +74,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the password reset token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -89,31 +83,31 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Enable or disable the Google Calendar integration', + type: 'boolean', }) - @CastToBoolean() CALENDAR_PROVIDER_GOOGLE_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Callback URL for Google Auth APIs', + type: 'string', }) AUTH_GOOGLE_APIS_CALLBACK_URL: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Enable or disable Google Single Sign-On (SSO)', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() AUTH_GOOGLE_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, isSensitive: true, description: 'Client ID for Google authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_GOOGLE_ENABLED) AUTH_GOOGLE_CLIENT_ID: string; @@ -121,8 +115,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.GoogleAuth, isSensitive: true, description: 'Client secret for Google authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_GOOGLE_ENABLED) AUTH_GOOGLE_CLIENT_SECRET: string; @@ -130,6 +124,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.GoogleAuth, isSensitive: true, description: 'Callback URL for Google authentication', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @ValidateIf((env) => env.AUTH_GOOGLE_ENABLED) @@ -138,25 +133,24 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Enable or disable the Gmail messaging integration', + type: 'boolean', }) - @CastToBoolean() MESSAGING_PROVIDER_GMAIL_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, description: 'Enable or disable Microsoft authentication', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() AUTH_MICROSOFT_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Client ID for Microsoft authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) AUTH_MICROSOFT_CLIENT_ID: string; @@ -164,8 +158,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Client secret for Microsoft authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) AUTH_MICROSOFT_CLIENT_SECRET: string; @@ -173,6 +167,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Callback URL for Microsoft authentication', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) @@ -182,6 +177,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Callback URL for Microsoft APIs', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) @@ -190,15 +186,15 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, description: 'Enable or disable the Microsoft messaging integration', + type: 'boolean', }) - @CastToBoolean() MESSAGING_PROVIDER_MICROSOFT_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, description: 'Enable or disable the Microsoft Calendar integration', + type: 'boolean', }) - @CastToBoolean() CALENDAR_PROVIDER_MICROSOFT_ENABLED = false; @ConfigVariablesMetadata({ @@ -206,14 +202,15 @@ export class ConfigVariables { isSensitive: true, description: 'Legacy variable to be deprecated when all API Keys expire. Replaced by APP_KEY', + type: 'string', }) @IsOptional() - @IsString() ACCESS_TOKEN_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the access token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -222,6 +219,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the refresh token is valid', + type: 'string', }) @IsOptional() REFRESH_TOKEN_EXPIRES_IN = '60d'; @@ -229,6 +227,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Cooldown period for refreshing tokens', + type: 'string', }) @IsDuration() @IsOptional() @@ -237,6 +236,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the login token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -245,6 +245,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the file token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -253,6 +254,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the invitation token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -261,51 +263,58 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the short-term token is valid', + type: 'string', }) SHORT_TERM_TOKEN_EXPIRES_IN = '5m'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Email address used as the sender for outgoing emails', + type: 'string', }) EMAIL_FROM_ADDRESS = 'noreply@yourdomain.com'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Email address used for system notifications', + type: 'string', }) EMAIL_SYSTEM_ADDRESS = 'system@yourdomain.com'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Name used in the From header for outgoing emails', + type: 'string', }) EMAIL_FROM_NAME = 'Felix from Twenty'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Email driver to use for sending emails', + type: 'enum', + options: Object.values(EmailDriver), }) EMAIL_DRIVER: EmailDriver = EmailDriver.Logger; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'SMTP host for sending emails', + type: 'string', }) EMAIL_SMTP_HOST: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Use unsecure connection for SMTP', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() EMAIL_SMTP_NO_TLS = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'SMTP port for sending emails', + type: 'number', }) @CastToPositiveNumber() EMAIL_SMTP_PORT = 587; @@ -313,6 +322,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'SMTP user for authentication', + type: 'string', }) EMAIL_SMTP_USER: string; @@ -320,28 +330,31 @@ export class ConfigVariables { group: ConfigVariablesGroup.EmailSettings, isSensitive: true, description: 'SMTP password for authentication', + type: 'string', }) EMAIL_SMTP_PASSWORD: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'Type of storage to use (local or S3)', + type: 'enum', + options: Object.values(StorageDriverType), }) - @IsEnum(StorageDriverType) @IsOptional() STORAGE_TYPE: StorageDriverType = StorageDriverType.Local; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'Local path for storage when using local storage type', + type: 'string', }) - @IsString() @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.Local) STORAGE_LOCAL_PATH = '.local-storage'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'S3 region for storage when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) @IsAWSRegion() @@ -350,17 +363,17 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'S3 bucket name for storage when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() STORAGE_S3_NAME: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'S3 endpoint for storage when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() @IsOptional() STORAGE_S3_ENDPOINT: string; @@ -369,9 +382,9 @@ export class ConfigVariables { isSensitive: true, description: 'S3 access key ID for authentication when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() @IsOptional() STORAGE_S3_ACCESS_KEY_ID: string; @@ -380,23 +393,25 @@ export class ConfigVariables { isSensitive: true, description: 'S3 secret access key for authentication when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() @IsOptional() STORAGE_S3_SECRET_ACCESS_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Type of serverless execution (local or Lambda)', + type: 'enum', + options: Object.values(ServerlessDriverType), }) - @IsEnum(ServerlessDriverType) @IsOptional() SERVERLESS_TYPE: ServerlessDriverType = ServerlessDriverType.Local; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Throttle limit for serverless function execution', + type: 'number', }) @CastToPositiveNumber() SERVERLESS_FUNCTION_EXEC_THROTTLE_LIMIT = 10; @@ -405,6 +420,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Time-to-live for serverless function execution throttle', + type: 'number', }) @CastToPositiveNumber() SERVERLESS_FUNCTION_EXEC_THROTTLE_TTL = 1000; @@ -412,6 +428,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Region for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) @IsAWSRegion() @@ -420,17 +437,17 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'IAM role for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() SERVERLESS_LAMBDA_ROLE: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Role to assume when hosting lambdas in dedicated AWS account', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() @IsOptional() SERVERLESS_LAMBDA_SUBHOSTING_ROLE?: string; @@ -438,9 +455,9 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerlessConfig, isSensitive: true, description: 'Access key ID for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() @IsOptional() SERVERLESS_LAMBDA_ACCESS_KEY_ID: string; @@ -448,24 +465,24 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerlessConfig, isSensitive: true, description: 'Secret access key for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() @IsOptional() SERVERLESS_LAMBDA_SECRET_ACCESS_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.AnalyticsConfig, description: 'Enable or disable analytics for telemetry', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() ANALYTICS_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.AnalyticsConfig, description: 'Clickhouse host for analytics', + type: 'string', }) @IsOptional() @IsUrl({ @@ -478,34 +495,32 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Enable or disable telemetry logging', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() TELEMETRY_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Enable or disable billing features', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_BILLING_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Link required for billing plan', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_PLAN_REQUIRED_LINK: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Duration of free trial with credit card in days', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @IsOptional() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) @@ -514,8 +529,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Duration of free trial without credit card in days', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @IsOptional() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) @@ -524,8 +539,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Amount of money in cents to trigger a billing threshold', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_SUBSCRIPTION_THRESHOLD_AMOUNT = 10000; @@ -533,8 +548,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Amount of credits for the free trial without credit card', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_FREE_WORKFLOW_CREDITS_FOR_TRIAL_PERIOD_WITHOUT_CREDIT_CARD = 5000; @@ -542,8 +557,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Amount of credits for the free trial with credit card', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_FREE_WORKFLOW_CREDITS_FOR_TRIAL_PERIOD_WITH_CREDIT_CARD = 10000; @@ -552,8 +567,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.BillingConfig, isSensitive: true, description: 'Stripe API key for billing', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_STRIPE_API_KEY: string; @@ -561,14 +576,15 @@ export class ConfigVariables { group: ConfigVariablesGroup.BillingConfig, isSensitive: true, description: 'Stripe webhook secret for billing', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_STRIPE_WEBHOOK_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Url for the frontend application', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @IsOptional() @@ -578,33 +594,33 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, description: 'Default subdomain for the frontend when multi-workspace is enabled', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_MULTIWORKSPACE_ENABLED) DEFAULT_SUBDOMAIN = 'app'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'ID for the Chrome extension', + type: 'string', }) - @IsString() @IsOptional() CHROME_EXTENSION_ID: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Enable or disable buffering for logs before sending', + type: 'boolean', }) - @CastToBoolean() - @IsBoolean() @IsOptional() LOGGER_IS_BUFFER_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Driver used for handling exceptions (Console or Sentry)', + type: 'enum', + options: Object.values(ExceptionHandlerDriver), }) - @IsEnum(ExceptionHandlerDriver) @IsOptional() EXCEPTION_HANDLER_DRIVER: ExceptionHandlerDriver = ExceptionHandlerDriver.Console; @@ -612,6 +628,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Levels of logging to be captured', + type: 'array', + options: ['log', 'error', 'warn'], }) @CastToLogLevelArray() @IsOptional() @@ -620,6 +638,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Metering, description: 'Driver used for collect metrics (OpenTelemetry or Console)', + type: 'array', + options: ['OpenTelemetry', 'Console'], }) @CastToMeterDriverArray() @IsOptional() @@ -628,6 +648,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Metering, description: 'Endpoint URL for the OpenTelemetry collector', + type: 'string', }) @IsOptional() OTLP_COLLECTOR_ENDPOINT_URL: string; @@ -635,47 +656,48 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Driver used for logging (only console for now)', + type: 'enum', + options: Object.values(LoggerDriverType), }) - @IsEnum(LoggerDriverType) @IsOptional() LOGGER_DRIVER: LoggerDriverType = LoggerDriverType.Console; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Data Source Name (DSN) for Sentry logging', + type: 'string', }) @ValidateIf( (env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry, ) - @IsString() SENTRY_DSN: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Front-end DSN for Sentry logging', + type: 'string', }) @ValidateIf( (env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry, ) - @IsString() SENTRY_FRONT_DSN: string; - @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Environment name for Sentry logging', + type: 'string', }) @ValidateIf( (env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry, ) - @IsString() @IsOptional() SENTRY_ENVIRONMENT: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.SupportChatConfig, description: 'Driver used for support chat integration', + type: 'enum', + options: Object.values(SupportDriver), }) - @IsEnum(SupportDriver) @IsOptional() SUPPORT_DRIVER: SupportDriver = SupportDriver.None; @@ -683,24 +705,26 @@ export class ConfigVariables { group: ConfigVariablesGroup.SupportChatConfig, isSensitive: true, description: 'Chat ID for the support front integration', + type: 'string', }) @ValidateIf((env) => env.SUPPORT_DRIVER === SupportDriver.Front) - @IsString() SUPPORT_FRONT_CHAT_ID: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.SupportChatConfig, isSensitive: true, description: 'HMAC key for the support front integration', + type: 'string', }) @ValidateIf((env) => env.SUPPORT_DRIVER === SupportDriver.Front) - @IsString() SUPPORT_FRONT_HMAC_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'Database connection URL', + type: 'string', + isEnvOnly: true, }) @IsDefined() @IsUrl({ @@ -715,24 +739,25 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, description: 'Allow connections to a database with self-signed certificates', + isEnvOnly: true, + type: 'boolean', }) - @CastToBoolean() - @IsBoolean() @IsOptional() PG_SSL_ALLOW_SELF_SIGNED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Enable configuration variables to be stored in the database', + isEnvOnly: true, + type: 'boolean', }) - @CastToBoolean() - @IsBoolean() @IsOptional() IS_CONFIG_VARIABLES_IN_DB_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Time-to-live for cache storage in seconds', + type: 'number', }) @CastToPositiveNumber() CACHE_STORAGE_TTL: number = 3600 * 24 * 7; @@ -741,6 +766,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'URL for cache storage (e.g., Redis connection URL)', + isEnvOnly: true, + type: 'string', }) @IsOptional() @IsUrl({ @@ -753,23 +780,24 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Node environment (development, production, etc.)', + type: 'enum', + options: Object.values(NodeEnvironment), }) - @IsEnum(NodeEnvironment) - @IsString() NODE_ENV: NodeEnvironment = NodeEnvironment.production; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Port for the node server', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @IsOptional() NODE_PORT = 3000; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Base URL for the server', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @IsOptional() @@ -779,22 +807,24 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'Secret key for the application', + isEnvOnly: true, + type: 'string', }) - @IsString() APP_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Maximum number of records affected by mutations', + type: 'number', }) @CastToPositiveNumber() @IsOptional() - @IsNumber() MUTATION_MAXIMUM_AFFECTED_RECORDS = 100; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Time-to-live for API rate limiting in milliseconds', + type: 'number', }) @CastToPositiveNumber() API_RATE_LIMITING_TTL = 100; @@ -803,6 +833,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.RateLimiting, description: 'Maximum number of requests allowed in the rate limiting window', + type: 'number', }) @CastToPositiveNumber() API_RATE_LIMITING_LIMIT = 500; @@ -810,8 +841,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.SSL, description: 'Path to the SSL key for enabling HTTPS in local development', + type: 'string', }) - @IsString() @IsOptional() SSL_KEY_PATH: string; @@ -819,8 +850,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.SSL, description: 'Path to the SSL certificate for enabling HTTPS in local development', + type: 'string', }) - @IsString() @IsOptional() SSL_CERT_PATH: string; @@ -828,30 +859,32 @@ export class ConfigVariables { group: ConfigVariablesGroup.CloudflareConfig, isSensitive: true, description: 'API key for Cloudflare integration', + type: 'string', }) - @IsString() @ValidateIf((env) => env.CLOUDFLARE_ZONE_ID) CLOUDFLARE_API_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.CloudflareConfig, description: 'Zone ID for Cloudflare integration', + type: 'string', }) - @IsString() @ValidateIf((env) => env.CLOUDFLARE_API_KEY) CLOUDFLARE_ZONE_ID: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Random string to validate queries from Cloudflare', + type: 'string', }) - @IsString() @IsOptional() CLOUDFLARE_WEBHOOK_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.LLM, description: 'Driver for the LLM chat model', + type: 'enum', + options: Object.values(LLMChatModelDriver), }) LLM_CHAT_MODEL_DRIVER: LLMChatModelDriver; @@ -859,6 +892,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.LLM, isSensitive: true, description: 'API key for OpenAI integration', + type: 'string', }) OPENAI_API_KEY: string; @@ -866,37 +900,40 @@ export class ConfigVariables { group: ConfigVariablesGroup.LLM, isSensitive: true, description: 'Secret key for Langfuse integration', + type: 'string', }) LANGFUSE_SECRET_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.LLM, description: 'Public key for Langfuse integration', + type: 'string', }) LANGFUSE_PUBLIC_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.LLM, description: 'Driver for LLM tracing', + type: 'enum', + options: Object.values(LLMTracingDriver), }) LLM_TRACING_DRIVER: LLMTracingDriver = LLMTracingDriver.Console; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Enable or disable multi-workspace support', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_MULTIWORKSPACE_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Number of inactive days before sending a deletion warning for workspaces. Used in the workspace deletion cron job to determine when to send warning emails.', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @IsStrictlyLowerThan('WORKSPACE_INACTIVE_DAYS_BEFORE_SOFT_DELETION', { message: '"WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION" should be strictly lower than "WORKSPACE_INACTIVE_DAYS_BEFORE_SOFT_DELETION"', @@ -906,9 +943,9 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Number of inactive days before soft deleting workspaces', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @IsStrictlyLowerThan('WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION', { message: '"WORKSPACE_INACTIVE_DAYS_BEFORE_SOFT_DELETION" should be strictly lower than "WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION"', @@ -918,24 +955,25 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Number of inactive days before deleting workspaces', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION = 21; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Maximum number of workspaces that can be deleted in a single execution', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @ValidateIf((env) => env.MAX_NUMBER_OF_WORKSPACES_DELETED_PER_EXECUTION > 0) MAX_NUMBER_OF_WORKSPACES_DELETED_PER_EXECUTION = 5; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Throttle limit for workflow execution', + type: 'number', }) @CastToPositiveNumber() WORKFLOW_EXEC_THROTTLE_LIMIT = 500; @@ -943,6 +981,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Time-to-live for workflow execution throttle in milliseconds', + type: 'number', }) @CastToPositiveNumber() WORKFLOW_EXEC_THROTTLE_TTL = 1000; @@ -950,8 +989,9 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.CaptchaConfig, description: 'Driver for captcha integration', + type: 'enum', + options: Object.values(CaptchaDriverType), }) - @IsEnum(CaptchaDriverType) @IsOptional() CAPTCHA_DRIVER?: CaptchaDriverType; @@ -959,8 +999,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.CaptchaConfig, isSensitive: true, description: 'Site key for captcha integration', + type: 'string', }) - @IsString() @IsOptional() CAPTCHA_SITE_KEY?: string; @@ -968,8 +1008,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.CaptchaConfig, isSensitive: true, description: 'Secret key for captcha integration', + type: 'string', }) - @IsString() @IsOptional() CAPTCHA_SECRET_KEY?: string; @@ -977,16 +1017,16 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'License key for the Enterprise version', + type: 'string', }) - @IsString() @IsOptional() ENTERPRISE_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Health monitoring time window in minutes', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @IsOptional() HEALTH_METRICS_TIME_WINDOW_IN_MINUTES = 5; @@ -994,15 +1034,15 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Enable or disable the attachment preview feature', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_ATTACHMENT_PREVIEW_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Twenty server version', + type: 'string', }) @IsOptionalOrEmptyString() @IsTwentySemVer() diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants.ts new file mode 100644 index 000000000..014a29c18 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants.ts @@ -0,0 +1,3 @@ +export const CONFIG_VARIABLES_INSTANCE_TOKEN = Symbol( + 'CONFIG_VARIABLES_INSTANCE_TOKEN', +); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants.ts new file mode 100644 index 000000000..7dcf53a62 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants.ts @@ -0,0 +1 @@ +export const CONFIG_VARIABLES_REFRESH_CRON_INTERVAL = '*/15 * * * * *'; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts new file mode 100644 index 000000000..e10a890c1 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts @@ -0,0 +1,472 @@ +import { LogLevel } from '@nestjs/common'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; +import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +// Mock configTransformers for type validation tests +jest.mock( + 'src/engine/core-modules/twenty-config/utils/config-transformers.util', + () => { + const originalModule = jest.requireActual( + 'src/engine/core-modules/twenty-config/utils/config-transformers.util', + ); + + return { + configTransformers: { + ...originalModule.configTransformers, + // These mocked versions can be overridden in specific tests + _mockedBoolean: jest.fn(), + _mockedNumber: jest.fn(), + _mockedString: jest.fn(), + }, + }; + }, +); + +describe('ConfigValueConverterService', () => { + let service: ConfigValueConverterService; + + beforeEach(async () => { + const mockConfigVariables = { + NODE_PORT: 3000, + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ConfigValueConverterService, + { + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, + useValue: mockConfigVariables, + }, + ], + }).compile(); + + service = module.get( + ConfigValueConverterService, + ); + }); + + describe('convertDbValueToAppValue', () => { + it('should convert string to boolean based on metadata', () => { + // Mock the metadata + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + AUTH_PASSWORD_ENABLED: { + type: 'boolean', + group: ConfigVariablesGroup.Other, + description: 'Enable or disable password authentication for users', + }, + }); + + expect( + service.convertDbValueToAppValue( + 'true', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + expect( + service.convertDbValueToAppValue( + 'True', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + expect( + service.convertDbValueToAppValue( + 'yes', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + expect( + service.convertDbValueToAppValue( + '1', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + + expect( + service.convertDbValueToAppValue( + 'false', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + expect( + service.convertDbValueToAppValue( + 'False', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + expect( + service.convertDbValueToAppValue( + 'no', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + expect( + service.convertDbValueToAppValue( + '0', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + }); + + it('should convert string to number based on metadata', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + NODE_PORT: { + type: 'number', + group: ConfigVariablesGroup.ServerConfig, + description: 'Port for the node server', + }, + }); + + expect( + service.convertDbValueToAppValue( + '42', + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBe(42); + expect( + service.convertDbValueToAppValue( + '3.14', + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBe(3.14); + + expect( + service.convertDbValueToAppValue( + 'not-a-number', + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBeUndefined(); + }); + + it('should convert string to array based on metadata', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Levels of logging to be captured', + }, + }); + + expect( + service.convertDbValueToAppValue( + 'log,error,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error', 'warn']); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Levels of logging to be captured', + }, + }); + expect( + service.convertDbValueToAppValue( + '["log","error","warn"]', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error', 'warn']); + }); + + it('should handle enum values as strings', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined); + + expect( + service.convertDbValueToAppValue( + 'development', + 'NODE_ENV' as keyof ConfigVariables, + ), + ).toBe('development'); + }); + + it('should handle various input types', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + AUTH_PASSWORD_ENABLED: { + type: 'boolean', + group: ConfigVariablesGroup.Other, + description: 'Enable or disable password authentication for users', + }, + }); + expect( + service.convertDbValueToAppValue( + true, + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + NODE_PORT: { + type: 'number', + group: ConfigVariablesGroup.ServerConfig, + description: 'Port for the node server', + }, + }); + expect( + service.convertDbValueToAppValue( + 42, + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBe(42); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Levels of logging to be captured', + }, + }); + expect( + service.convertDbValueToAppValue( + ['log', 'error'] as LogLevel[], + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error']); + }); + + it('should fall back to default value approach when no metadata', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined); + + expect( + service.convertDbValueToAppValue( + '42', + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBe(42); + }); + + it('should handle null and undefined values', () => { + expect( + service.convertDbValueToAppValue( + null, + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBeUndefined(); + + expect( + service.convertDbValueToAppValue( + undefined, + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBeUndefined(); + }); + + it('should throw error if boolean converter returns non-boolean', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + AUTH_PASSWORD_ENABLED: { + type: 'boolean', + group: ConfigVariablesGroup.Other, + description: 'Test boolean', + }, + }); + + const originalBoolean = configTransformers.boolean; + + configTransformers.boolean = jest + .fn() + .mockImplementation(() => 'not-a-boolean'); + + expect(() => { + service.convertDbValueToAppValue( + 'true', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ); + }).toThrow(/Expected boolean for key AUTH_PASSWORD_ENABLED/); + + configTransformers.boolean = originalBoolean; + }); + + it('should throw error if number converter returns non-number', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + NODE_PORT: { + type: 'number', + group: ConfigVariablesGroup.ServerConfig, + description: 'Test number', + }, + }); + + const originalNumber = configTransformers.number; + + configTransformers.number = jest + .fn() + .mockImplementation(() => 'not-a-number'); + + expect(() => { + service.convertDbValueToAppValue( + '42', + 'NODE_PORT' as keyof ConfigVariables, + ); + }).toThrow(/Expected number for key NODE_PORT/); + + configTransformers.number = originalNumber; + }); + + it('should throw error if string converter returns non-string', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + EMAIL_FROM_ADDRESS: { + type: 'string', + group: ConfigVariablesGroup.EmailSettings, + description: 'Test string', + }, + }); + + const originalString = configTransformers.string; + + configTransformers.string = jest.fn().mockImplementation(() => 42); + + expect(() => { + service.convertDbValueToAppValue( + 'test@example.com', + 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables, + ); + }).toThrow(/Expected string for key EMAIL_FROM_ADDRESS/); + + configTransformers.string = originalString; + }); + + it('should throw error if array conversion produces non-array', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Test array', + }, + }); + + const convertToArraySpy = jest + .spyOn( + service as any, // Cast to any to access private method + 'convertToArray', + ) + .mockReturnValueOnce('not-an-array'); + + expect(() => { + service.convertDbValueToAppValue( + 'log,error,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ); + }).toThrow(/Expected array for key LOG_LEVELS/); + + convertToArraySpy.mockRestore(); + }); + + it('should handle array with option validation', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Test array with options', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'log,error,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error', 'warn']); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Test array with options', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'log,invalid,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'warn']); + }); + + it('should properly handle enum with options', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVEL: { + type: 'enum', + group: ConfigVariablesGroup.Logging, + description: 'Test enum', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'error', + 'LOG_LEVEL' as keyof ConfigVariables, + ), + ).toBe('error'); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVEL: { + type: 'enum', + group: ConfigVariablesGroup.Logging, + description: 'Test enum', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'invalid', + 'LOG_LEVEL' as keyof ConfigVariables, + ), + ).toBeUndefined(); + }); + }); + + describe('convertAppValueToDbValue', () => { + it('should handle primitive types directly', () => { + expect(service.convertAppValueToDbValue('string-value' as any)).toBe( + 'string-value', + ); + expect(service.convertAppValueToDbValue(42 as any)).toBe(42); + expect(service.convertAppValueToDbValue(true as any)).toBe(true); + expect(service.convertAppValueToDbValue(undefined as any)).toBe(null); + }); + + it('should handle arrays', () => { + const array = ['log', 'error', 'warn'] as LogLevel[]; + + expect(service.convertAppValueToDbValue(array as any)).toEqual(array); + }); + + it('should handle objects', () => { + const obj = { key: 'value' }; + + expect(service.convertAppValueToDbValue(obj as any)).toEqual(obj); + }); + + it('should convert null to null', () => { + expect(service.convertAppValueToDbValue(null as any)).toBe(null); + }); + + it('should throw error for unsupported types', () => { + const symbol = Symbol('test'); + + expect(() => { + service.convertAppValueToDbValue(symbol as any); + }).toThrow(/Cannot convert value of type symbol/); + }); + + it('should handle serialization errors', () => { + // Create an object with circular reference + const circular: any = {}; + + circular.self = circular; + + expect(() => { + service.convertAppValueToDbValue(circular as any); + }).toThrow(/Failed to serialize object value/); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts new file mode 100644 index 000000000..33bb46f58 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -0,0 +1,213 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; +import { ConfigVariablesMetadataMap } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; +import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; +import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +@Injectable() +export class ConfigValueConverterService { + private readonly logger = new Logger(ConfigValueConverterService.name); + + constructor( + @Inject(CONFIG_VARIABLES_INSTANCE_TOKEN) + private readonly configVariables: ConfigVariables, + ) {} + + convertDbValueToAppValue( + dbValue: unknown, + key: T, + ): ConfigVariables[T] | undefined { + if (dbValue === null || dbValue === undefined) { + return undefined; + } + + const metadata = this.getConfigVariableMetadata(key); + const configType = metadata?.type || this.inferTypeFromValue(key); + const options = metadata?.options; + + try { + switch (configType) { + case 'boolean': { + const result = configTransformers.boolean(dbValue); + + if (result !== undefined && typeof result !== 'boolean') { + throw new Error( + `Expected boolean for key ${key}, got ${typeof result}`, + ); + } + + return result as ConfigVariables[T]; + } + + case 'number': { + const result = configTransformers.number(dbValue); + + if (result !== undefined && typeof result !== 'number') { + throw new Error( + `Expected number for key ${key}, got ${typeof result}`, + ); + } + + return result as ConfigVariables[T]; + } + + case 'string': { + const result = configTransformers.string(dbValue); + + if (result !== undefined && typeof result !== 'string') { + throw new Error( + `Expected string for key ${key}, got ${typeof result}`, + ); + } + + return result as ConfigVariables[T]; + } + + case 'array': { + const result = this.convertToArray(dbValue, options); + + if (result !== undefined && !Array.isArray(result)) { + throw new Error( + `Expected array for key ${key}, got ${typeof result}`, + ); + } + + return result as ConfigVariables[T]; + } + + case 'enum': { + const result = this.convertToEnum(dbValue, options); + + return result as ConfigVariables[T]; + } + + default: + return dbValue as ConfigVariables[T]; + } + } catch (error) { + throw new Error( + `Failed to convert ${key as string} to app value: ${(error as Error).message}`, + ); + } + } + + convertAppValueToDbValue( + appValue: ConfigVariables[T] | null | undefined, + ): unknown { + if (appValue === undefined || appValue === null) { + return null; + } + + if ( + typeof appValue === 'string' || + typeof appValue === 'number' || + typeof appValue === 'boolean' + ) { + return appValue; + } + + if (Array.isArray(appValue)) { + return appValue; + } + + if (typeof appValue === 'object') { + try { + return JSON.parse(JSON.stringify(appValue)); + } catch (error) { + throw new Error( + `Failed to serialize object value: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + throw new Error( + `Cannot convert value of type ${typeof appValue} to storage format`, + ); + } + + private convertToArray( + value: unknown, + options?: ConfigVariableOptions, + ): unknown[] { + if (Array.isArray(value)) { + return this.validateArrayAgainstOptions(value, options); + } + + if (typeof value === 'string') { + try { + const parsedArray = JSON.parse(value); + + if (Array.isArray(parsedArray)) { + return this.validateArrayAgainstOptions(parsedArray, options); + } + } catch { + const splitArray = value.split(',').map((item) => item.trim()); + + return this.validateArrayAgainstOptions(splitArray, options); + } + } + + return this.validateArrayAgainstOptions([value], options); + } + + private validateArrayAgainstOptions( + array: unknown[], + options?: ConfigVariableOptions, + ): unknown[] { + if (!options || !Array.isArray(options) || options.length === 0) { + return array; + } + + return array.filter((item) => { + const included = options.includes(item as string); + + if (!included) { + this.logger.debug( + `Filtered out array item '${String(item)}' not in allowed options`, + ); + } + + return included; + }); + } + + private convertToEnum( + value: unknown, + options?: ConfigVariableOptions, + ): unknown | undefined { + if (!options || !Array.isArray(options) || options.length === 0) { + return value; + } + + if (options.includes(value as string)) { + return value; + } + + return undefined; + } + + private getConfigVariableMetadata(key: T) { + const allMetadata = TypedReflect.getMetadata( + 'config-variables', + ConfigVariables.prototype.constructor, + ) as ConfigVariablesMetadataMap | undefined; + + return allMetadata?.[key as string]; + } + + private inferTypeFromValue( + key: T, + ): ConfigVariableType { + const defaultValue = this.configVariables[key]; + + if (typeof defaultValue === 'boolean') return 'boolean'; + if (typeof defaultValue === 'number') return 'number'; + if (Array.isArray(defaultValue)) return 'array'; + + return 'string'; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts deleted file mode 100644 index 8b1942a10..000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Transform } from 'class-transformer'; - -export const CastToBoolean = () => - Transform(({ value }: { value: string }) => toBoolean(value)); - -const toBoolean = (value: any) => { - if (typeof value === 'boolean') { - return value; - } - if (['true', 'on', 'yes', '1'].includes(value.toLowerCase())) { - return true; - } - if (['false', 'off', 'no', '0'].includes(value.toLowerCase())) { - return false; - } - - return undefined; -}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-string-array.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-string-array.decorator.ts deleted file mode 100644 index b70af4e4a..000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-string-array.decorator.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Transform } from 'class-transformer'; - -export const CastToStringArray = () => - Transform(({ value }: { value: string }) => toStringArray(value)); - -const toStringArray = (value: any) => { - if (typeof value === 'string') { - return value.split(',').map((item) => item.trim()); - } - - return undefined; -}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts index 5f84c9da8..914d79558 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts @@ -1,12 +1,22 @@ -import { registerDecorator, ValidationOptions } from 'class-validator'; +import { + IsOptional, + registerDecorator, + ValidationOptions, +} from 'class-validator'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; +import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; +import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; +import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util'; import { TypedReflect } from 'src/utils/typed-reflect'; export interface ConfigVariablesMetadataOptions { group: ConfigVariablesGroup; description: string; isSensitive?: boolean; + isEnvOnly?: boolean; + type?: ConfigVariableType; + options?: ConfigVariableOptions; } export type ConfigVariablesMetadataMap = { @@ -30,6 +40,26 @@ export function ConfigVariablesMetadata( target.constructor, ); + const propertyDescriptor = Object.getOwnPropertyDescriptor( + target.constructor.prototype, + propertyKey, + ); + const hasDefaultValue = + propertyDescriptor && propertyDescriptor.value !== undefined; + + if (!hasDefaultValue) { + IsOptional()(target, propertyKey); + } + + if (options.type) { + applyBasicValidators( + options.type, + target, + propertyKey.toString(), + options.options, + ); + } + registerDecorator({ name: propertyKey.toString(), target: target.constructor, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts new file mode 100644 index 000000000..9398e033a --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -0,0 +1,374 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; + +jest.mock( + 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util', + () => ({ + isEnvOnlyConfigVar: jest.fn(), + }), +); + +const CONFIG_PASSWORD_KEY = 'AUTH_PASSWORD_ENABLED'; +const CONFIG_EMAIL_KEY = 'EMAIL_FROM_ADDRESS'; +const CONFIG_ENV_ONLY_KEY = 'ENV_ONLY_VAR'; +const CONFIG_PORT_KEY = 'NODE_PORT'; + +class TestDatabaseConfigDriver extends DatabaseConfigDriver { + // Expose the protected/private property for testing + public get testAllPossibleConfigKeys(): Array { + return this['allPossibleConfigKeys']; + } + + // Override Object.keys usage in constructor with our test keys + constructor( + configCache: ConfigCacheService, + configStorage: ConfigStorageService, + ) { + super(configCache, configStorage); + + Object.defineProperty(this, 'allPossibleConfigKeys', { + value: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY, CONFIG_PORT_KEY], + writable: false, + configurable: true, + }); + } +} + +describe('DatabaseConfigDriver', () => { + let driver: TestDatabaseConfigDriver; + let configCache: ConfigCacheService; + let configStorage: ConfigStorageService; + + beforeEach(async () => { + (isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => { + return key === CONFIG_ENV_ONLY_KEY; + }); + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + { + provide: DatabaseConfigDriver, + useClass: TestDatabaseConfigDriver, + }, + { + provide: ConfigCacheService, + useValue: { + get: jest.fn(), + set: jest.fn(), + clear: jest.fn(), + clearAll: jest.fn(), + isKeyKnownMissing: jest.fn(), + markKeyAsMissing: jest.fn(), + getCacheInfo: jest.fn(), + getAllKeys: jest.fn(), + }, + }, + { + provide: ConfigStorageService, + useValue: { + get: jest.fn(), + set: jest.fn(), + loadAll: jest.fn(), + }, + }, + ], + }).compile(); + + driver = module.get( + DatabaseConfigDriver, + ) as TestDatabaseConfigDriver; + configCache = module.get(ConfigCacheService); + configStorage = module.get(ConfigStorageService); + + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + jest.useRealTimers(); + }); + + it('should be defined', () => { + expect(driver).toBeDefined(); + }); + + describe('initialization', () => { + it('should have allPossibleConfigKeys properly set', () => { + expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_PASSWORD_KEY); + expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_EMAIL_KEY); + expect(driver.testAllPossibleConfigKeys).not.toContain( + CONFIG_ENV_ONLY_KEY, + ); + }); + + it('should initialize successfully with DB values and mark missing keys', async () => { + const configVars = new Map(); + + configVars.set(CONFIG_PASSWORD_KEY, true); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); + + await driver.onModuleInit(); + + expect(configStorage.loadAll).toHaveBeenCalled(); + + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + ); + + expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith( + CONFIG_ENV_ONLY_KEY, + ); + }); + + it('should handle initialization failure gracefully', async () => { + const error = new Error('DB error'); + + jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); + jest.spyOn(driver['logger'], 'error').mockImplementation(); + + // Should not throw because we're handling errors internally now + await driver.onModuleInit(); + + expect(driver['logger'].error).toHaveBeenCalled(); + expect(configStorage.loadAll).toHaveBeenCalled(); + }); + }); + + describe('get', () => { + it('should return cached value when available', async () => { + const cachedValue = true; + + jest.spyOn(configCache, 'get').mockReturnValue(cachedValue); + + const result = driver.get(CONFIG_PASSWORD_KEY); + + expect(result).toBe(cachedValue); + expect(configCache.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + }); + + it('should return undefined when value is not in cache', async () => { + jest.spyOn(configCache, 'get').mockReturnValue(undefined); + + const result = driver.get(CONFIG_PASSWORD_KEY); + + expect(result).toBeUndefined(); + expect(configCache.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + }); + + it('should handle different config variable types correctly', () => { + const stringValue = 'test@example.com'; + const booleanValue = true; + const numberValue = 3000; + + jest.spyOn(configCache, 'get').mockImplementation((key) => { + switch (key) { + case CONFIG_EMAIL_KEY: + return stringValue; + case CONFIG_PASSWORD_KEY: + return booleanValue; + case CONFIG_PORT_KEY: + return numberValue; + default: + return undefined; + } + }); + + expect(driver.get(CONFIG_EMAIL_KEY)).toBe(stringValue); + expect(driver.get(CONFIG_PASSWORD_KEY)).toBe(booleanValue); + expect(driver.get(CONFIG_PORT_KEY)).toBe(numberValue); + }); + }); + + describe('update', () => { + beforeEach(async () => { + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); + }); + + it('should update config in storage and cache', async () => { + const value = true; + + await driver.update(CONFIG_PASSWORD_KEY, value); + + expect(configStorage.set).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + value, + ); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value); + }); + + it('should throw error when updating env-only variable', async () => { + const value = true; + + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); + + await expect(driver.update(CONFIG_PASSWORD_KEY, value)).rejects.toThrow(); + }); + }); + + describe('fetchAndCacheConfigVariable', () => { + it('should refresh config variable from storage', async () => { + const value = true; + + jest.spyOn(configStorage, 'get').mockResolvedValue(value); + + await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); + + expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value); + }); + + it('should mark key as missing when value is undefined', async () => { + jest.spyOn(configStorage, 'get').mockResolvedValue(undefined); + + await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); + + expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + ); + expect(configCache.set).not.toHaveBeenCalled(); + }); + + it('should mark key as missing when storage fetch fails', async () => { + const error = new Error('Storage error'); + + jest.spyOn(configStorage, 'get').mockRejectedValue(error); + const loggerSpy = jest + .spyOn(driver['logger'], 'error') + .mockImplementation(); + + await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); + + expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + ); + expect(loggerSpy).toHaveBeenCalledWith( + expect.stringContaining('Failed to fetch config'), + error, + ); + }); + }); + + describe('cache operations', () => { + it('should return cache info', () => { + const cacheInfo = { + foundConfigValues: 2, + knownMissingKeys: 1, + cacheKeys: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY], + }; + + jest.spyOn(configCache, 'getCacheInfo').mockReturnValue(cacheInfo); + + const result = driver.getCacheInfo(); + + expect(result).toEqual(cacheInfo); + }); + }); + + describe('refreshAllCache', () => { + it('should load all config values from DB', async () => { + const dbValues = new Map(); + + dbValues.set(CONFIG_PASSWORD_KEY, true); + dbValues.set(CONFIG_EMAIL_KEY, 'test@example.com'); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); + + await driver.refreshAllCache(); + + expect(configStorage.loadAll).toHaveBeenCalled(); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + expect(configCache.set).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + 'test@example.com', + ); + }); + + it('should not affect env-only variables when found in DB', async () => { + const dbValues = new Map(); + + dbValues.set(CONFIG_PASSWORD_KEY, true); + dbValues.set(CONFIG_ENV_ONLY_KEY, 'env-value'); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); + + await driver.refreshAllCache(); + + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + + expect(configCache.set).not.toHaveBeenCalledWith( + CONFIG_ENV_ONLY_KEY, + 'env-value', + ); + }); + + it('should mark keys as missing when not found in DB', async () => { + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + + await driver.refreshAllCache(); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + ); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + ); + + expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith( + CONFIG_ENV_ONLY_KEY, + ); + }); + + it('should properly handle mix of found and missing keys', async () => { + const dbValues = new Map(); + + dbValues.set(CONFIG_PASSWORD_KEY, true); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); + + await driver.refreshAllCache(); + + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + ); + }); + + it('should handle errors gracefully and verify cache remains unchanged', async () => { + const error = new Error('Database error'); + + jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); + jest.spyOn(driver['logger'], 'error').mockImplementation(); + + const mockCacheState = new Map(); + + mockCacheState.set(CONFIG_PASSWORD_KEY, false); + jest + .spyOn(configCache, 'getAllKeys') + .mockReturnValue([CONFIG_PASSWORD_KEY]); + jest + .spyOn(configCache, 'get') + .mockImplementation((key) => mockCacheState.get(key)); + + await driver.refreshAllCache(); + + expect(driver['logger'].error).toHaveBeenCalled(); + expect(configStorage.loadAll).toHaveBeenCalled(); + + expect(configCache.set).not.toHaveBeenCalled(); + expect(configCache.markKeyAsMissing).not.toHaveBeenCalled(); + expect(configCache.clear).not.toHaveBeenCalled(); + expect(configCache.clearAll).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts new file mode 100644 index 000000000..1efc5fe90 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts @@ -0,0 +1,110 @@ +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; + +describe('EnvironmentConfigDriver', () => { + let driver: EnvironmentConfigDriver; + let configService: ConfigService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + EnvironmentConfigDriver, + { + provide: ConfigService, + useValue: { + get: jest.fn(), + }, + }, + { + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, + useValue: new ConfigVariables(), + }, + ], + }).compile(); + + driver = module.get(EnvironmentConfigDriver); + configService = module.get(ConfigService); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should be defined', () => { + expect(driver).toBeDefined(); + }); + + describe('get', () => { + it('should return value from config service when available', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const expectedValue = true; + const defaultValue = new ConfigVariables()[key]; + + jest.spyOn(configService, 'get').mockReturnValue(expectedValue); + + const result = driver.get(key); + + expect(result).toBe(expectedValue); + expect(configService.get).toHaveBeenCalledWith(key, defaultValue); + }); + + it('should return default value when config service returns undefined', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const defaultValue = new ConfigVariables()[key]; + + jest + .spyOn(configService, 'get') + .mockImplementation((_, defaultVal) => defaultVal); + + const result = driver.get(key); + + expect(result).toBe(defaultValue); + expect(configService.get).toHaveBeenCalledWith(key, defaultValue); + }); + + it('should handle different config variable types', () => { + const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const numberKey = 'NODE_PORT' as keyof ConfigVariables; + + const defaultValues = new ConfigVariables(); + + jest + .spyOn(configService, 'get') + .mockImplementation((key: keyof ConfigVariables) => { + switch (key) { + case booleanKey: + return true; + case stringKey: + return 'test@example.com'; + case numberKey: + return 3000; + default: + return undefined; + } + }); + + expect(driver.get(booleanKey)).toBe(true); + expect(configService.get).toHaveBeenCalledWith( + booleanKey, + defaultValues[booleanKey], + ); + + expect(driver.get(stringKey)).toBe('test@example.com'); + expect(configService.get).toHaveBeenCalledWith( + stringKey, + defaultValues[stringKey], + ); + + expect(driver.get(numberKey)).toBe(3000); + expect(configService.get).toHaveBeenCalledWith( + numberKey, + defaultValues[numberKey], + ); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts new file mode 100644 index 000000000..8c8be4831 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -0,0 +1,193 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; + +import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface'; + +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_REFRESH_CRON_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; + +@Injectable() +export class DatabaseConfigDriver + implements DatabaseConfigDriverInterface, OnModuleInit +{ + private readonly logger = new Logger(DatabaseConfigDriver.name); + private readonly allPossibleConfigKeys: Array; + + constructor( + private readonly configCache: ConfigCacheService, + private readonly configStorage: ConfigStorageService, + ) { + const allKeys = Object.keys(new ConfigVariables()) as Array< + keyof ConfigVariables + >; + + this.allPossibleConfigKeys = allKeys.filter( + (key) => !isEnvOnlyConfigVar(key), + ); + + this.logger.debug( + '[INIT] Database config driver created, monitoring keys: ' + + this.allPossibleConfigKeys.length, + ); + } + + async onModuleInit(): Promise { + try { + this.logger.log('[INIT] Loading initial config variables from database'); + const loadedCount = await this.loadAllConfigVarsFromDb(); + + this.logger.log( + `[INIT] Config variables loaded: ${loadedCount} values found, ${this.allPossibleConfigKeys.length - loadedCount} missing`, + ); + } catch (error) { + this.logger.error( + '[INIT] Failed to load config variables from database, falling back to environment variables', + error instanceof Error ? error.stack : error, + ); + // Don't rethrow to allow the application to continue + // The driver's cache will be empty but the service will fall back to env vars + } + } + + get(key: T): ConfigVariables[T] | undefined { + return this.configCache.get(key); + } + + async update( + key: T, + value: ConfigVariables[T], + ): Promise { + if (isEnvOnlyConfigVar(key)) { + throw new Error( + `Cannot update environment-only variable: ${key as string}`, + ); + } + + try { + await this.configStorage.set(key, value); + this.configCache.set(key, value); + this.logger.debug( + `[UPDATE] Config variable ${key as string} updated successfully`, + ); + } catch (error) { + this.logger.error( + `[UPDATE] Failed to update config variable ${key as string}`, + error, + ); + throw error; + } + } + + async fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise { + try { + const value = await this.configStorage.get(key); + + if (value !== undefined) { + this.configCache.set(key, value); + this.logger.debug( + `[FETCH] Config variable ${key as string} loaded from database`, + ); + } else { + this.configCache.markKeyAsMissing(key); + this.logger.debug( + `[FETCH] Config variable ${key as string} not found in database, marked as missing`, + ); + } + } catch (error) { + this.logger.error( + `[FETCH] Failed to fetch config variable ${key as string} from database`, + error, + ); + this.configCache.markKeyAsMissing(key); + } + } + + getCacheInfo(): { + foundConfigValues: number; + knownMissingKeys: number; + cacheKeys: string[]; + } { + return this.configCache.getCacheInfo(); + } + + private async loadAllConfigVarsFromDb(): Promise { + try { + this.logger.debug('[LOAD] Fetching all config variables from database'); + const configVars = await this.configStorage.loadAll(); + + this.logger.debug( + `[LOAD] Processing ${this.allPossibleConfigKeys.length} possible config variables`, + ); + + for (const [key, value] of configVars.entries()) { + this.configCache.set(key, value); + } + + for (const key of this.allPossibleConfigKeys) { + if (!configVars.has(key)) { + this.configCache.markKeyAsMissing(key); + } + } + + const missingKeysCount = + this.allPossibleConfigKeys.length - configVars.size; + + this.logger.debug( + `[LOAD] Cached ${configVars.size} config variables, marked ${missingKeysCount} keys as missing`, + ); + + return configVars.size; + } catch (error) { + this.logger.error( + '[LOAD] Failed to load config variables from database', + error, + ); + throw error; + } + } + + /** + * Refreshes all database-backed config variables. + * This method runs on a schedule and fetches all configs in one database query, + * then updates the cache with fresh values. + */ + @Cron(CONFIG_VARIABLES_REFRESH_CRON_INTERVAL) + async refreshAllCache(): Promise { + try { + this.logger.debug( + '[REFRESH] Starting scheduled refresh of config variables', + ); + + const dbValues = await this.configStorage.loadAll(); + + this.logger.debug( + `[REFRESH] Processing ${this.allPossibleConfigKeys.length} possible config variables`, + ); + + for (const [key, value] of dbValues.entries()) { + if (!isEnvOnlyConfigVar(key)) { + this.configCache.set(key, value); + } + } + + for (const key of this.allPossibleConfigKeys) { + if (!dbValues.has(key)) { + this.configCache.markKeyAsMissing(key); + } + } + + const missingKeysCount = + this.allPossibleConfigKeys.length - dbValues.size; + + this.logger.log( + `[REFRESH] Config variables refreshed: ${dbValues.size} values updated, ${missingKeysCount} marked as missing`, + ); + } catch (error) { + this.logger.error('[REFRESH] Failed to refresh config variables', error); + // Error is caught and logged but not rethrown to prevent the cron job from crashing + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts new file mode 100644 index 000000000..d22af8caf --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts @@ -0,0 +1,35 @@ +import { DynamicModule, Module } from '@nestjs/common'; +import { ScheduleModule } from '@nestjs/schedule'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; + +@Module({}) +export class DatabaseConfigModule { + static forRoot(): DynamicModule { + return { + module: DatabaseConfigModule, + imports: [ + TypeOrmModule.forFeature([KeyValuePair], 'core'), + ScheduleModule.forRoot(), + ], + providers: [ + DatabaseConfigDriver, + ConfigCacheService, + ConfigStorageService, + ConfigValueConverterService, + { + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, + useValue: new ConfigVariables(), + }, + ], + exports: [DatabaseConfigDriver], + }; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts new file mode 100644 index 000000000..b87381f57 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts @@ -0,0 +1,21 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; + +@Injectable() +export class EnvironmentConfigDriver { + constructor( + private readonly configService: ConfigService, + @Inject(CONFIG_VARIABLES_INSTANCE_TOKEN) + private readonly defaultConfigVariables: ConfigVariables, + ) {} + + get(key: T): ConfigVariables[T] { + return this.configService.get( + key, + this.defaultConfigVariables[key], + ); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts new file mode 100644 index 000000000..a61a80a19 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -0,0 +1,40 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +/** + * Interface for drivers that support database-backed configuration + * with caching capabilities + */ +export interface DatabaseConfigDriverInterface { + /** + * Get a configuration value from cache + * Returns undefined if not in cache + */ + get(key: T): ConfigVariables[T] | undefined; + + /** + * Update a configuration value in the database and cache + */ + update( + key: T, + value: ConfigVariables[T], + ): Promise; + + /** + * Fetch and cache a specific configuration from its source + */ + fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise; + + /** + * Refreshes all entries in the config cache + */ + refreshAllCache(): Promise; + + /** + * Get information about the cache state + */ + getCacheInfo(): { + foundConfigValues: number; + knownMissingKeys: number; + cacheKeys: string[]; + }; +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-source.enum.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-source.enum.ts new file mode 100644 index 000000000..2dbc408ae --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-source.enum.ts @@ -0,0 +1,5 @@ +export enum ConfigSource { + ENVIRONMENT = 'ENVIRONMENT', + DATABASE = 'DATABASE', + DEFAULT = 'DEFAULT', +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts new file mode 100644 index 000000000..e0c45a920 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts @@ -0,0 +1,488 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; + +import { DeleteResult, IsNull, Repository } from 'typeorm'; + +import { + KeyValuePair, + KeyValuePairType, +} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; +import { User } from 'src/engine/core-modules/user/user.entity'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; + +describe('ConfigStorageService', () => { + let service: ConfigStorageService; + let keyValuePairRepository: Repository; + let configValueConverter: ConfigValueConverterService; + + const createMockKeyValuePair = ( + key: string, + value: string, + ): KeyValuePair => ({ + id: '1', + key, + value: value as unknown as JSON, + type: KeyValuePairType.CONFIG_VARIABLE, + userId: null, + workspaceId: null, + user: null as unknown as User, + workspace: null as unknown as Workspace, + createdAt: new Date(), + updatedAt: new Date(), + textValueDeprecated: null, + deletedAt: null, + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ConfigStorageService, + { + provide: ConfigValueConverterService, + useValue: { + convertDbValueToAppValue: jest.fn(), + convertAppValueToDbValue: jest.fn(), + }, + }, + ConfigVariables, + { + provide: getRepositoryToken(KeyValuePair, 'core'), + useValue: { + findOne: jest.fn(), + find: jest.fn(), + update: jest.fn(), + insert: jest.fn(), + delete: jest.fn(), + }, + }, + ], + }).compile(); + + service = module.get(ConfigStorageService); + keyValuePairRepository = module.get>( + getRepositoryToken(KeyValuePair, 'core'), + ); + configValueConverter = module.get( + ConfigValueConverterService, + ); + + jest.clearAllMocks(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('get', () => { + it('should return undefined when key not found', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null); + + const result = await service.get(key); + + expect(result).toBeUndefined(); + expect(keyValuePairRepository.findOne).toHaveBeenCalledWith({ + where: { + type: KeyValuePairType.CONFIG_VARIABLE, + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + }, + }); + }); + + it('should return converted value when key found', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const storedValue = 'true'; + const convertedValue = true; + + const mockRecord = createMockKeyValuePair(key as string, storedValue); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockReturnValue(convertedValue); + + const result = await service.get(key); + + expect(result).toBe(convertedValue); + expect( + configValueConverter.convertDbValueToAppValue, + ).toHaveBeenCalledWith(storedValue, key); + }); + + it('should handle conversion errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Conversion error'); + + const mockRecord = createMockKeyValuePair(key as string, 'invalid'); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation(() => { + throw error; + }); + + await expect(service.get(key)).rejects.toThrow('Conversion error'); + }); + }); + + describe('set', () => { + it('should update existing record', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const storedValue = 'true'; + + const mockRecord = createMockKeyValuePair(key as string, 'false'); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + ( + configValueConverter.convertAppValueToDbValue as jest.Mock + ).mockReturnValue(storedValue); + + await service.set(key, value); + + expect(keyValuePairRepository.update).toHaveBeenCalledWith( + { id: '1' }, + { value: storedValue }, + ); + }); + + it('should insert new record', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const storedValue = 'true'; + + jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null); + + ( + configValueConverter.convertAppValueToDbValue as jest.Mock + ).mockReturnValue(storedValue); + + await service.set(key, value); + + expect(keyValuePairRepository.insert).toHaveBeenCalledWith({ + key: key as string, + value: storedValue, + userId: null, + workspaceId: null, + type: KeyValuePairType.CONFIG_VARIABLE, + }); + }); + + it('should handle conversion errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const error = new Error('Conversion error'); + + ( + configValueConverter.convertAppValueToDbValue as jest.Mock + ).mockImplementation(() => { + throw error; + }); + + await expect(service.set(key, value)).rejects.toThrow('Conversion error'); + }); + }); + + describe('delete', () => { + it('should delete record', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + await service.delete(key); + + expect(keyValuePairRepository.delete).toHaveBeenCalledWith({ + type: KeyValuePairType.CONFIG_VARIABLE, + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + }); + }); + + it('should handle delete errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Delete error'); + + jest.spyOn(keyValuePairRepository, 'delete').mockRejectedValue(error); + + await expect(service.delete(key)).rejects.toThrow('Delete error'); + }); + }); + + describe('loadAll', () => { + it('should load and convert all config variables', async () => { + const configVars: KeyValuePair[] = [ + createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), + createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), + ]; + + jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation((value, key) => { + if (key === 'AUTH_PASSWORD_ENABLED') return true; + if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com'; + + return value; + }); + + const result = await service.loadAll(); + + expect(result.size).toBe(2); + expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe( + true, + ); + expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( + 'test@example.com', + ); + }); + + it('should skip invalid values but continue processing', async () => { + const configVars: KeyValuePair[] = [ + createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'invalid'), + createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), + ]; + + jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); + + (configValueConverter.convertDbValueToAppValue as jest.Mock) + .mockImplementationOnce(() => { + throw new Error('Invalid value'); + }) + .mockImplementationOnce((value) => value); + + const result = await service.loadAll(); + + expect(result.size).toBe(1); + expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( + 'test@example.com', + ); + }); + + it('should handle find errors', async () => { + const error = new Error('Find error'); + + jest.spyOn(keyValuePairRepository, 'find').mockRejectedValue(error); + + await expect(service.loadAll()).rejects.toThrow('Find error'); + }); + + describe('Null Value Handling', () => { + it('should handle null values in loadAll', async () => { + const configVars: KeyValuePair[] = [ + { + ...createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), + value: null as unknown as JSON, + }, + createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), + ]; + + jest + .spyOn(keyValuePairRepository, 'find') + .mockResolvedValue(configVars); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation((value) => { + if (value === null) throw new Error('Null value'); + + return value; + }); + + const result = await service.loadAll(); + + expect(result.size).toBe(1); + expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( + 'test@example.com', + ); + expect( + configValueConverter.convertDbValueToAppValue, + ).toHaveBeenCalledTimes(1); // Only called for non-null value + }); + }); + }); + + describe('Edge Cases and Additional Scenarios', () => { + describe('Type Safety', () => { + it('should enforce correct types for boolean config variables', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const invalidValue = 'not-a-boolean'; + + const mockRecord = createMockKeyValuePair(key as string, invalidValue); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation(() => { + throw new Error('Invalid boolean value'); + }); + + await expect(service.get(key)).rejects.toThrow('Invalid boolean value'); + }); + + it('should enforce correct types for string config variables', async () => { + const key = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const invalidValue = '123'; // Not a valid email + + const mockRecord = createMockKeyValuePair(key as string, invalidValue); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation(() => { + throw new Error('Invalid string value'); + }); + + await expect(service.get(key)).rejects.toThrow('Invalid string value'); + }); + }); + + describe('Concurrent Operations', () => { + it('should handle concurrent get/set operations', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const initialValue = true; + const newValue = false; + + const initialRecord = createMockKeyValuePair(key as string, 'true'); + const updatedRecord = createMockKeyValuePair(key as string, 'false'); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValueOnce(initialRecord) + .mockResolvedValueOnce(initialRecord) + .mockResolvedValueOnce(updatedRecord); + + (configValueConverter.convertDbValueToAppValue as jest.Mock) + .mockReturnValueOnce(initialValue) + .mockReturnValueOnce(newValue); + + ( + configValueConverter.convertAppValueToDbValue as jest.Mock + ).mockReturnValueOnce('false'); + + const firstGet = service.get(key); + const setOperation = service.set(key, newValue); + const secondGet = service.get(key); + + const [firstResult, , secondResult] = await Promise.all([ + firstGet, + setOperation, + secondGet, + ]); + + expect(firstResult).toBe(initialValue); + expect(secondResult).toBe(newValue); + }); + + it('should handle concurrent delete operations', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + jest + .spyOn(keyValuePairRepository, 'delete') + .mockResolvedValueOnce({ affected: 1 } as DeleteResult) + .mockResolvedValueOnce({ affected: 0 } as DeleteResult); + + const firstDelete = service.delete(key); + const secondDelete = service.delete(key); + + await Promise.all([firstDelete, secondDelete]); + + expect(keyValuePairRepository.delete).toHaveBeenCalledTimes(2); + }); + }); + + describe('Database Connection Issues', () => { + it('should handle database connection failures in get', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Database connection failed'); + + jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); + + await expect(service.get(key)).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database connection failures in set', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const error = new Error('Database connection failed'); + + ( + configValueConverter.convertAppValueToDbValue as jest.Mock + ).mockReturnValue('true'); + jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); + + await expect(service.set(key, value)).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database connection failures in delete', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Database connection failed'); + + jest.spyOn(keyValuePairRepository, 'delete').mockRejectedValue(error); + + await expect(service.delete(key)).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database connection failures in loadAll', async () => { + const error = new Error('Database connection failed'); + + jest.spyOn(keyValuePairRepository, 'find').mockRejectedValue(error); + + await expect(service.loadAll()).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database operation timeouts', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Database operation timed out'); + + let rejectPromise: ((error: Error) => void) | undefined; + const timeoutPromise = new Promise((_, reject) => { + rejectPromise = reject; + }); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockReturnValue(timeoutPromise); + + const promise = service.get(key); + + // Simulate timeout by rejecting the promise + if (!rejectPromise) { + throw new Error('Reject function not assigned'); + } + rejectPromise(error); + + await expect(promise).rejects.toThrow('Database operation timed out'); + }); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts new file mode 100644 index 000000000..0254d9f17 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -0,0 +1,165 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { FindOptionsWhere, IsNull, Repository } from 'typeorm'; + +import { + KeyValuePair, + KeyValuePairType, +} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; + +import { ConfigStorageInterface } from './interfaces/config-storage.interface'; + +@Injectable() +export class ConfigStorageService implements ConfigStorageInterface { + private readonly logger = new Logger(ConfigStorageService.name); + + constructor( + @InjectRepository(KeyValuePair, 'core') + private readonly keyValuePairRepository: Repository, + private readonly configValueConverter: ConfigValueConverterService, + ) {} + + private getConfigVariableWhereClause( + key?: string, + ): FindOptionsWhere { + return { + type: KeyValuePairType.CONFIG_VARIABLE, + ...(key ? { key } : {}), + userId: IsNull(), + workspaceId: IsNull(), + }; + } + + async get( + key: T, + ): Promise { + try { + const result = await this.keyValuePairRepository.findOne({ + where: this.getConfigVariableWhereClause(key as string), + }); + + if (result === null) { + return undefined; + } + + try { + this.logger.debug( + `Fetching config for ${key as string} in database: ${result?.value}`, + ); + + return this.configValueConverter.convertDbValueToAppValue( + result.value, + key, + ); + } catch (error) { + this.logger.error( + `Failed to convert value to app type for key ${key as string}`, + error, + ); + throw error; + } + } catch (error) { + this.logger.error(`Failed to get config for ${key as string}`, error); + throw error; + } + } + + async set( + key: T, + value: ConfigVariables[T], + ): Promise { + try { + let processedValue; + + try { + processedValue = + this.configValueConverter.convertAppValueToDbValue(value); + } catch (error) { + this.logger.error( + `Failed to convert value to storage type for key ${key as string}`, + error, + ); + throw error; + } + + const existingRecord = await this.keyValuePairRepository.findOne({ + where: this.getConfigVariableWhereClause(key as string), + }); + + if (existingRecord) { + await this.keyValuePairRepository.update( + { id: existingRecord.id }, + { value: processedValue }, + ); + } else { + await this.keyValuePairRepository.insert({ + key: key as string, + value: processedValue, + userId: null, + workspaceId: null, + type: KeyValuePairType.CONFIG_VARIABLE, + }); + } + } catch (error) { + this.logger.error(`Failed to set config for ${key as string}`, error); + throw error; + } + } + + async delete(key: T): Promise { + try { + await this.keyValuePairRepository.delete( + this.getConfigVariableWhereClause(key as string), + ); + } catch (error) { + this.logger.error(`Failed to delete config for ${key as string}`, error); + throw error; + } + } + + async loadAll(): Promise< + Map + > { + try { + const configVars = await this.keyValuePairRepository.find({ + where: this.getConfigVariableWhereClause(), + }); + + const result = new Map< + keyof ConfigVariables, + ConfigVariables[keyof ConfigVariables] + >(); + + for (const configVar of configVars) { + if (configVar.value !== null) { + const key = configVar.key as keyof ConfigVariables; + + try { + const value = this.configValueConverter.convertDbValueToAppValue( + configVar.value, + key, + ); + + if (value !== undefined) { + result.set(key, value); + } + } catch (error) { + this.logger.error( + `Failed to convert value to app type for key ${key as string}`, + error, + ); + continue; + } + } + } + + return result; + } catch (error) { + this.logger.error('Failed to load all config variables', error); + throw error; + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts new file mode 100644 index 000000000..cab1fa21d --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts @@ -0,0 +1,18 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +export interface ConfigStorageInterface { + get( + key: T, + ): Promise; + + set( + key: T, + value: ConfigVariables[T], + ): Promise; + + delete(key: T): Promise; + + loadAll(): Promise< + Map + >; +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index 2083625d8..d650e8d62 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -1,21 +1,48 @@ -import { Global, Module } from '@nestjs/common'; +import { DynamicModule, Global, Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; -import { validate } from 'src/engine/core-modules/twenty-config/config-variables'; +import { + ConfigVariables, + validate, +} from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; +import { DatabaseConfigModule } from 'src/engine/core-modules/twenty-config/drivers/database-config.module'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { ConfigurableModuleClass } from 'src/engine/core-modules/twenty-config/twenty-config.module-definition'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; @Global() -@Module({ - imports: [ - ConfigModule.forRoot({ - isGlobal: true, - expandVariables: true, - validate, - envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', - }), - ], - providers: [TwentyConfigService], - exports: [TwentyConfigService], -}) -export class TwentyConfigModule extends ConfigurableModuleClass {} +@Module({}) +export class TwentyConfigModule extends ConfigurableModuleClass { + static forRoot(): DynamicModule { + const isConfigVariablesInDbEnabled = + process.env.IS_CONFIG_VARIABLES_IN_DB_ENABLED === 'true'; + + const imports = [ + ConfigModule.forRoot({ + isGlobal: true, + expandVariables: true, + validate, + envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', + }), + ]; + + if (isConfigVariablesInDbEnabled) { + imports.push(DatabaseConfigModule.forRoot()); + } + + return { + module: TwentyConfigModule, + imports, + providers: [ + TwentyConfigService, + EnvironmentConfigDriver, + { + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, + useValue: new ConfigVariables(), + }, + ], + exports: [TwentyConfigService], + }; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index 977c67c78..f8e2ab94b 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -2,81 +2,470 @@ import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; +import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +jest.mock('src/utils/typed-reflect', () => ({ + TypedReflect: { + getMetadata: jest.fn(), + defineMetadata: jest.fn(), + }, +})); + +jest.mock( + 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config', + () => ({ + CONFIG_VARIABLES_MASKING_CONFIG: { + SENSITIVE_VAR: { + strategy: 'LAST_N_CHARS', + chars: 5, + }, + }, + }), +); + +jest.mock( + 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util', + () => ({ + isEnvOnlyConfigVar: jest.fn(), + }), +); + +type TwentyConfigServicePrivateProps = { + isDatabaseDriverActive: boolean; +}; + +const mockConfigVarMetadata = { + TEST_VAR: { + group: ConfigVariablesGroup.GoogleAuth, + description: 'Test variable', + isEnvOnly: false, + }, + ENV_ONLY_VAR: { + group: ConfigVariablesGroup.StorageConfig, + description: 'Environment only variable', + isEnvOnly: true, + }, + SENSITIVE_VAR: { + group: ConfigVariablesGroup.Logging, + description: 'Sensitive variable', + isSensitive: true, + }, +}; + +// Setup with database driver +const setupTestModule = async (isDatabaseConfigEnabled = true) => { + const configServiceMock = { + get: jest.fn().mockImplementation((key) => { + if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { + return isDatabaseConfigEnabled ? 'true' : 'false'; + } + + return undefined; + }), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + TwentyConfigService, + { + provide: DatabaseConfigDriver, + useValue: { + get: jest.fn(), + update: jest.fn(), + getCacheInfo: jest.fn(), + }, + }, + { + provide: EnvironmentConfigDriver, + useValue: { + get: jest.fn().mockImplementation((key) => { + return configServiceMock.get(key); + }), + }, + }, + { + provide: ConfigService, + useValue: configServiceMock, + }, + ], + }).compile(); + + return { + service: module.get(TwentyConfigService), + databaseConfigDriver: + module.get(DatabaseConfigDriver), + environmentConfigDriver: module.get( + EnvironmentConfigDriver, + ), + configService: module.get(ConfigService), + }; +}; + +// Setup without database driver +const setupTestModuleWithoutDb = async () => { + const configServiceMock = { + get: jest.fn().mockImplementation((key) => { + if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { + return 'false'; + } + + return undefined; + }), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + TwentyConfigService, + { + provide: EnvironmentConfigDriver, + useValue: { + get: jest.fn().mockImplementation((key) => { + return configServiceMock.get(key); + }), + }, + }, + { + provide: ConfigService, + useValue: configServiceMock, + }, + ], + }).compile(); + + return { + service: module.get(TwentyConfigService), + environmentConfigDriver: module.get( + EnvironmentConfigDriver, + ), + configService: module.get(ConfigService), + }; +}; + +const setPrivateProps = ( + service: TwentyConfigService, + props: Partial, +) => { + Object.entries(props).forEach(([key, value]) => { + Object.defineProperty(service, key, { + value, + writable: true, + }); + }); +}; describe('TwentyConfigService', () => { let service: TwentyConfigService; - let configService: ConfigService; + let databaseConfigDriver: DatabaseConfigDriver; + let environmentConfigDriver: EnvironmentConfigDriver; beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - TwentyConfigService, - { - provide: ConfigService, - useValue: { - get: jest.fn(), - }, - }, - ], - }).compile(); + const testModule = await setupTestModule(true); - service = module.get(TwentyConfigService); - configService = module.get(ConfigService); + service = testModule.service; + databaseConfigDriver = testModule.databaseConfigDriver; + environmentConfigDriver = testModule.environmentConfigDriver; - Reflect.defineMetadata('config-variables', {}, ConfigVariables); + (TypedReflect.getMetadata as jest.Mock).mockReturnValue( + mockConfigVarMetadata, + ); + }); + + afterEach(() => { + jest.clearAllMocks(); }); it('should be defined', () => { expect(service).toBeDefined(); }); - describe('getAll()', () => { - it('should return empty object when no config variables are defined', () => { - const result = service.getAll(); + describe('constructor', () => { + it('should set isDatabaseDriverActive to false when database config is disabled', async () => { + const { service, environmentConfigDriver } = + await setupTestModuleWithoutDb(); - expect(result).toEqual({}); + expect(environmentConfigDriver.get).toHaveBeenCalledWith( + 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', + ); + + expect(service.getCacheInfo().usingDatabaseDriver).toBe(false); }); - it('should return config variables with their metadata', () => { - const mockMetadata = { - TEST_VAR: { - title: 'Test Var', - description: 'Test Description', - }, - }; + it('should set isDatabaseDriverActive to true when database config is enabled and driver is available', async () => { + const { service, environmentConfigDriver } = await setupTestModule(true); - Reflect.defineMetadata('config-variables', mockMetadata, ConfigVariables); + expect(environmentConfigDriver.get).toHaveBeenCalledWith( + 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', + ); - jest.spyOn(configService, 'get').mockReturnValue('test-value'); + expect(service.getCacheInfo().usingDatabaseDriver).toBe(true); + }); + }); + + describe('get', () => { + const key = 'TEST_VAR' as keyof ConfigVariables; + const expectedValue = 'test value'; + + beforeEach(() => { + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); + }); + + it('should use environment driver for environment-only variables', () => { + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(expectedValue); + + const result = service.get(key); + + expect(result).toBe(expectedValue); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); + }); + + it('should return undefined when key does not exist in any driver', () => { + const nonExistentKey = 'NON_EXISTENT_KEY' as keyof ConfigVariables; + + jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(undefined); + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(undefined); + setPrivateProps(service, { isDatabaseDriverActive: true }); + + const result = service.get(nonExistentKey); + + expect(result).toBeUndefined(); + expect(databaseConfigDriver.get).toHaveBeenCalledWith(nonExistentKey); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(nonExistentKey); + }); + + it('should use database driver when isDatabaseDriverActive is true and value is found', () => { + jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue); + setPrivateProps(service, { isDatabaseDriverActive: true }); + + jest.clearAllMocks(); + + const result = service.get(key); + + expect(result).toBe(expectedValue); + expect(databaseConfigDriver.get).toHaveBeenCalledWith(key); + expect(environmentConfigDriver.get).not.toHaveBeenCalled(); + }); + + it('should fall back to environment driver when database driver is active but value is not found', () => { + const envValue = 'env value'; + + jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(undefined); + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(envValue); + setPrivateProps(service, { isDatabaseDriverActive: true }); + + const result = service.get(key); + + expect(result).toBe(envValue); + expect(databaseConfigDriver.get).toHaveBeenCalledWith(key); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); + }); + + it('should use environment driver when isDatabaseDriverActive is false', () => { + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(expectedValue); + setPrivateProps(service, { isDatabaseDriverActive: false }); + + const result = service.get(key); + + expect(result).toBe(expectedValue); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); + expect(databaseConfigDriver.get).not.toHaveBeenCalled(); + }); + }); + + describe('update', () => { + it('should throw error when database driver is not active', async () => { + setPrivateProps(service, { isDatabaseDriverActive: false }); + + await expect( + service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow( + 'Database configuration is disabled or unavailable, cannot update configuration', + ); + }); + + it('should throw error when updating environment-only variable', async () => { + setPrivateProps(service, { isDatabaseDriverActive: true }); + (TypedReflect.getMetadata as jest.Mock).mockReturnValue({ + ENV_ONLY_VAR: { isEnvOnly: true }, + }); + + await expect( + service.update('ENV_ONLY_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow( + 'Cannot update environment-only variable: ENV_ONLY_VAR', + ); + }); + + it('should update config when database driver is active', async () => { + const key = 'TEST_VAR' as keyof ConfigVariables; + const newValue = 'new value'; + + setPrivateProps(service, { isDatabaseDriverActive: true }); + jest.spyOn(databaseConfigDriver, 'update').mockResolvedValue(undefined); + + await service.update(key, newValue); + + expect(databaseConfigDriver.update).toHaveBeenCalledWith(key, newValue); + }); + + it('should propagate errors from database driver', async () => { + const error = new Error('Database error'); + + setPrivateProps(service, { isDatabaseDriverActive: true }); + jest.spyOn(databaseConfigDriver, 'update').mockRejectedValue(error); + + await expect( + service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow(error); + }); + }); + + describe('getMetadata', () => { + it('should return metadata for a config variable', () => { + const result = service.getMetadata('TEST_VAR' as keyof ConfigVariables); + + expect(result).toEqual(mockConfigVarMetadata.TEST_VAR); + }); + + it('should return undefined when metadata does not exist', () => { + const result = service.getMetadata( + 'UNKNOWN_VAR' as keyof ConfigVariables, + ); + + expect(result).toBeUndefined(); + }); + }); + + describe('getAll', () => { + const setupDriverMocks = () => { + jest + .spyOn(environmentConfigDriver, 'get') + .mockImplementation((key: keyof ConfigVariables) => { + const keyStr = String(key); + const values = { + TEST_VAR: 'env test value', + ENV_ONLY_VAR: 'env only value', + SENSITIVE_VAR: 'sensitive_data_123', + }; + + return values[keyStr] || undefined; + }); + + jest + .spyOn(databaseConfigDriver, 'get') + .mockImplementation((key: keyof ConfigVariables) => { + const keyStr = String(key); + + if (mockConfigVarMetadata[keyStr]?.isEnvOnly) { + return environmentConfigDriver.get(key); + } + const values = { + TEST_VAR: 'db test value', + SENSITIVE_VAR: 'sensitive_data_123', + }; + + return values[keyStr] || undefined; + }); + }; + + beforeEach(() => { + setupDriverMocks(); + }); + + it('should return all config variables with environment source when database driver is not active', () => { + setPrivateProps(service, { + isDatabaseDriverActive: false, + }); const result = service.getAll(); expect(result).toEqual({ TEST_VAR: { - value: 'test-value', - metadata: mockMetadata.TEST_VAR, + value: 'env test value', + metadata: mockConfigVarMetadata.TEST_VAR, + source: ConfigSource.ENVIRONMENT, + }, + ENV_ONLY_VAR: { + value: 'env only value', + metadata: mockConfigVarMetadata.ENV_ONLY_VAR, + source: ConfigSource.ENVIRONMENT, + }, + SENSITIVE_VAR: { + value: expect.any(String), + metadata: mockConfigVarMetadata.SENSITIVE_VAR, + source: ConfigSource.ENVIRONMENT, }, }); + + expect(result.SENSITIVE_VAR.value).toBe('********a_123'); }); - it('should mask sensitive data according to masking config', () => { - const mockMetadata = { - APP_SECRET: { - title: 'App Secret', - description: 'Application secret key', - sensitive: true, - }, - }; - - Reflect.defineMetadata('config-variables', mockMetadata, ConfigVariables); - - jest.spyOn(configService, 'get').mockReturnValue('super-secret-value'); + it('should return config variables with database source when database driver is active', () => { + setPrivateProps(service, { + isDatabaseDriverActive: true, + }); const result = service.getAll(); - expect(result.APP_SECRET.value).not.toBe('super-secret-value'); - expect(result.APP_SECRET.value).toMatch(/^\*+[a-zA-Z0-9]{5}$/); + expect(result.TEST_VAR).toEqual({ + value: 'db test value', + metadata: mockConfigVarMetadata.TEST_VAR, + source: ConfigSource.DATABASE, + }); + + expect(result.ENV_ONLY_VAR).toEqual({ + value: 'env only value', + metadata: mockConfigVarMetadata.ENV_ONLY_VAR, + source: ConfigSource.ENVIRONMENT, + }); + + expect(result.SENSITIVE_VAR).toEqual({ + value: '********a_123', + metadata: mockConfigVarMetadata.SENSITIVE_VAR, + source: ConfigSource.DATABASE, + }); + }); + }); + + describe('getCacheInfo', () => { + it('should return basic info when database driver is not active', () => { + setPrivateProps(service, { + isDatabaseDriverActive: false, + }); + + const result = service.getCacheInfo(); + + expect(result).toEqual({ + usingDatabaseDriver: false, + }); + }); + + it('should return cache stats when database driver is active', () => { + const cacheStats = { + foundConfigValues: 2, + knownMissingKeys: 1, + cacheKeys: ['TEST_VAR', 'SENSITIVE_VAR'], + }; + + setPrivateProps(service, { + isDatabaseDriverActive: true, + }); + + jest + .spyOn(databaseConfigDriver, 'getCacheInfo') + .mockReturnValue(cacheStats); + + const result = service.getCacheInfo(); + + expect(result).toEqual({ + usingDatabaseDriver: true, + cacheStats, + }); }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 6dd527991..b5ed1783f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -1,22 +1,107 @@ -import { Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; +import { Injectable, Logger, Optional } from '@nestjs/common'; + +import { isString } from 'class-validator'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config'; import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesMaskingStrategies } from 'src/engine/core-modules/twenty-config/enums/config-variables-masking-strategies.enum'; import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty-config/utils/config-variable-mask-sensitive-data.util'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; import { TypedReflect } from 'src/utils/typed-reflect'; @Injectable() export class TwentyConfigService { - constructor(private readonly configService: ConfigService) {} + private readonly logger = new Logger(TwentyConfigService.name); + private readonly isDatabaseDriverActive: boolean; + + constructor( + private readonly environmentConfigDriver: EnvironmentConfigDriver, + @Optional() private readonly databaseConfigDriver: DatabaseConfigDriver, + ) { + const isConfigVariablesInDbEnabled = this.environmentConfigDriver.get( + 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', + ); + + this.isDatabaseDriverActive = + isConfigVariablesInDbEnabled && !!this.databaseConfigDriver; + + this.logger.log( + `Database configuration is ${isConfigVariablesInDbEnabled ? 'enabled' : 'disabled'}`, + ); + + if (isConfigVariablesInDbEnabled && !this.databaseConfigDriver) { + this.logger.warn( + 'Database config is enabled but driver is not available. Using environment variables only.', + ); + } + + if (this.isDatabaseDriverActive) { + this.logger.log('Using database configuration driver'); + // The database driver will load config variables asynchronously via its onModuleInit lifecycle hook + // In the meantime, we'll use the environment driver -- fallback + } else { + this.logger.log('Using environment variables only for configuration'); + } + } get(key: T): ConfigVariables[T] { - return this.configService.get( - key, - new ConfigVariables()[key], - ); + if (isEnvOnlyConfigVar(key)) { + return this.environmentConfigDriver.get(key); + } + + if (this.isDatabaseDriverActive) { + const cachedValueFromDb = this.databaseConfigDriver.get(key); + + if (cachedValueFromDb !== undefined) { + return cachedValueFromDb; + } + + return this.environmentConfigDriver.get(key); + } + + return this.environmentConfigDriver.get(key); + } + + async update( + key: T, + value: ConfigVariables[T], + ): Promise { + if (!this.isDatabaseDriverActive) { + throw new Error( + 'Database configuration is disabled or unavailable, cannot update configuration', + ); + } + + const metadata = + TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; + const envMetadata = metadata[key]; + + if (envMetadata?.isEnvOnly) { + throw new Error( + `Cannot update environment-only variable: ${key as string}`, + ); + } + + try { + await this.databaseConfigDriver.update(key, value); + this.logger.debug(`Updated config variable: ${key as string}`); + } catch (error) { + this.logger.error(`Failed to update config for ${key as string}`, error); + throw error; + } + } + + getMetadata( + key: keyof ConfigVariables, + ): ConfigVariablesMetadataOptions | undefined { + const metadata = + TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; + + return metadata[key]; } getAll(): Record< @@ -24,6 +109,7 @@ export class TwentyConfigService { { value: ConfigVariables[keyof ConfigVariables]; metadata: ConfigVariablesMetadataOptions; + source: ConfigSource; } > { const result: Record< @@ -31,6 +117,7 @@ export class TwentyConfigService { { value: ConfigVariables[keyof ConfigVariables]; metadata: ConfigVariablesMetadataOptions; + source: ConfigSource; } > = {}; @@ -39,12 +126,23 @@ export class TwentyConfigService { TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; Object.entries(metadata).forEach(([key, envMetadata]) => { - let value = - this.configService.get(key) ?? - configVars[key as keyof ConfigVariables] ?? - ''; + let value = this.get(key as keyof ConfigVariables) ?? ''; + let source = ConfigSource.ENVIRONMENT; - if (typeof value === 'string' && key in CONFIG_VARIABLES_MASKING_CONFIG) { + if (!this.isDatabaseDriverActive || envMetadata.isEnvOnly) { + if (value === configVars[key as keyof ConfigVariables]) { + source = ConfigSource.DEFAULT; + } + } else { + const dbValue = value; + + source = + dbValue !== configVars[key as keyof ConfigVariables] + ? ConfigSource.DATABASE + : ConfigSource.DEFAULT; + } + + if (isString(value) && key in CONFIG_VARIABLES_MASKING_CONFIG) { const varMaskingConfig = CONFIG_VARIABLES_MASKING_CONFIG[ key as keyof typeof CONFIG_VARIABLES_MASKING_CONFIG @@ -65,9 +163,32 @@ export class TwentyConfigService { result[key] = { value, metadata: envMetadata, + source, }; }); return result; } + + getCacheInfo(): { + usingDatabaseDriver: boolean; + cacheStats?: { + foundConfigValues: number; + knownMissingKeys: number; + cacheKeys: string[]; + }; + } { + const result = { + usingDatabaseDriver: this.isDatabaseDriverActive, + }; + + if (this.isDatabaseDriverActive) { + return { + ...result, + cacheStats: this.databaseConfigDriver.getCacheInfo(), + }; + } + + return result; + } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-options.type.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-options.type.ts new file mode 100644 index 000000000..067ebdc14 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-options.type.ts @@ -0,0 +1,3 @@ +export type ConfigVariableOptions = + | readonly (string | number | boolean)[] + | Record; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-type.type.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-type.type.ts new file mode 100644 index 000000000..0313be6ed --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-type.type.ts @@ -0,0 +1,6 @@ +export type ConfigVariableType = + | 'boolean' + | 'number' + | 'array' + | 'string' + | 'enum'; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts new file mode 100644 index 000000000..ed0f390b6 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts @@ -0,0 +1,161 @@ +import { Transform } from 'class-transformer'; +import { + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, +} from 'class-validator'; + +import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; + +jest.mock('class-transformer', () => ({ + Transform: jest.fn(), +})); + +jest.mock('class-validator', () => ({ + IsBoolean: jest.fn().mockReturnValue(jest.fn()), + IsNumber: jest.fn().mockReturnValue(jest.fn()), + IsString: jest.fn().mockReturnValue(jest.fn()), + IsEnum: jest.fn().mockReturnValue(jest.fn()), + IsArray: jest.fn().mockReturnValue(jest.fn()), +})); + +jest.mock( + 'src/engine/core-modules/twenty-config/utils/config-transformers.util', + () => ({ + configTransformers: { + boolean: jest.fn(), + number: jest.fn(), + }, + }), +); + +describe('applyBasicValidators', () => { + const mockTarget = {}; + const mockPropertyKey = 'testProperty'; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('boolean type', () => { + it('should apply boolean transformers and validators', () => { + let capturedTransformFn; + + (Transform as jest.Mock).mockImplementation((transformFn) => { + capturedTransformFn = transformFn; + + return jest.fn(); + }); + + applyBasicValidators('boolean', mockTarget, mockPropertyKey); + + expect(Transform).toHaveBeenCalled(); + expect(IsBoolean).toHaveBeenCalled(); + + const transformFn = capturedTransformFn; + const mockTransformParams = { value: 'true' }; + + (configTransformers.boolean as jest.Mock).mockReturnValueOnce(true); + const result1 = transformFn(mockTransformParams); + + expect(configTransformers.boolean).toHaveBeenCalledWith('true'); + expect(result1).toBe(true); + + (configTransformers.boolean as jest.Mock).mockReturnValueOnce(undefined); + const result2 = transformFn(mockTransformParams); + + expect(result2).toBe('true'); + }); + }); + + describe('number type', () => { + it('should apply number transformers and validators', () => { + let capturedTransformFn; + + (Transform as jest.Mock).mockImplementation((transformFn) => { + capturedTransformFn = transformFn; + + return jest.fn(); + }); + + applyBasicValidators('number', mockTarget, mockPropertyKey); + + expect(Transform).toHaveBeenCalled(); + expect(IsNumber).toHaveBeenCalled(); + + const transformFn = capturedTransformFn; + const mockTransformParams = { value: '42' }; + + (configTransformers.number as jest.Mock).mockReturnValueOnce(42); + const result1 = transformFn(mockTransformParams); + + expect(configTransformers.number).toHaveBeenCalledWith('42'); + expect(result1).toBe(42); + + (configTransformers.number as jest.Mock).mockReturnValueOnce(undefined); + const result2 = transformFn(mockTransformParams); + + expect(result2).toBe('42'); + }); + }); + + describe('string type', () => { + it('should apply string validator', () => { + applyBasicValidators('string', mockTarget, mockPropertyKey); + + expect(IsString).toHaveBeenCalled(); + expect(Transform).not.toHaveBeenCalled(); // String doesn't need a transform + }); + }); + + describe('enum type', () => { + it('should apply enum validator with string array options', () => { + const enumOptions = ['option1', 'option2', 'option3']; + + applyBasicValidators('enum', mockTarget, mockPropertyKey, enumOptions); + + expect(IsEnum).toHaveBeenCalledWith(enumOptions); + expect(Transform).not.toHaveBeenCalled(); // Enum doesn't need a transform + }); + + it('should apply enum validator with enum object options', () => { + enum TestEnum { + Option1 = 'value1', + Option2 = 'value2', + Option3 = 'value3', + } + + applyBasicValidators('enum', mockTarget, mockPropertyKey, TestEnum); + + expect(IsEnum).toHaveBeenCalledWith(TestEnum); + expect(Transform).not.toHaveBeenCalled(); // Enum doesn't need a transform + }); + + it('should not apply enum validator without options', () => { + applyBasicValidators('enum', mockTarget, mockPropertyKey); + + expect(IsEnum).not.toHaveBeenCalled(); + expect(Transform).not.toHaveBeenCalled(); + }); + }); + + describe('array type', () => { + it('should apply array validator', () => { + applyBasicValidators('array', mockTarget, mockPropertyKey); + + expect(IsArray).toHaveBeenCalled(); + expect(Transform).not.toHaveBeenCalled(); // Array doesn't need a transform + }); + }); + + describe('unsupported type', () => { + it('should throw error for unsupported types', () => { + expect(() => { + applyBasicValidators('unsupported' as any, mockTarget, mockPropertyKey); + }).toThrow('Unsupported config variable type: unsupported'); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/config-transformers.util.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/config-transformers.util.spec.ts new file mode 100644 index 000000000..ee43dc027 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/config-transformers.util.spec.ts @@ -0,0 +1,100 @@ +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; + +describe('configTransformers', () => { + describe('boolean', () => { + it('should handle true values correctly', () => { + expect(configTransformers.boolean(true)).toBe(true); + expect(configTransformers.boolean('true')).toBe(true); + expect(configTransformers.boolean('True')).toBe(true); + expect(configTransformers.boolean('yes')).toBe(true); + expect(configTransformers.boolean('on')).toBe(true); + expect(configTransformers.boolean('1')).toBe(true); + expect(configTransformers.boolean(1)).toBe(true); + }); + + it('should handle false values correctly', () => { + expect(configTransformers.boolean(false)).toBe(false); + expect(configTransformers.boolean('false')).toBe(false); + expect(configTransformers.boolean('False')).toBe(false); + expect(configTransformers.boolean('no')).toBe(false); + expect(configTransformers.boolean('off')).toBe(false); + expect(configTransformers.boolean('0')).toBe(false); + expect(configTransformers.boolean(0)).toBe(false); + }); + + it('should return undefined for invalid values', () => { + expect(configTransformers.boolean('invalid')).toBeUndefined(); + expect(configTransformers.boolean('random_string')).toBeUndefined(); + expect(configTransformers.boolean({})).toBeUndefined(); + expect(configTransformers.boolean([])).toBeUndefined(); + }); + + it('should handle null and undefined', () => { + expect(configTransformers.boolean(null)).toBeUndefined(); + expect(configTransformers.boolean(undefined)).toBeUndefined(); + }); + }); + + describe('number', () => { + it('should handle valid number values', () => { + expect(configTransformers.number(42)).toBe(42); + expect(configTransformers.number('42')).toBe(42); + expect(configTransformers.number('-42')).toBe(-42); + expect(configTransformers.number('3.14')).toBe(3.14); + expect(configTransformers.number('0')).toBe(0); + }); + + it('should handle boolean values', () => { + expect(configTransformers.number(true)).toBe(1); + expect(configTransformers.number(false)).toBe(0); + }); + + it('should return undefined for invalid values', () => { + expect(configTransformers.number('invalid')).toBeUndefined(); + expect(configTransformers.number('forty-two')).toBeUndefined(); + expect(configTransformers.number({})).toBeUndefined(); + expect(configTransformers.number([])).toBeUndefined(); + }); + + it('should handle null and undefined', () => { + expect(configTransformers.number(null)).toBeUndefined(); + expect(configTransformers.number(undefined)).toBeUndefined(); + }); + }); + + describe('string', () => { + it('should handle string values', () => { + expect(configTransformers.string('test')).toBe('test'); + expect(configTransformers.string('')).toBe(''); + }); + + it('should convert numbers to strings', () => { + expect(configTransformers.string(42)).toBe('42'); + expect(configTransformers.string(0)).toBe('0'); + expect(configTransformers.string(3.14)).toBe('3.14'); + }); + + it('should convert booleans to strings', () => { + expect(configTransformers.string(true)).toBe('true'); + expect(configTransformers.string(false)).toBe('false'); + }); + + it('should convert arrays and objects to JSON strings', () => { + expect(configTransformers.string(['a', 'b', 'c'])).toBe('["a","b","c"]'); + expect(configTransformers.string({ a: 1, b: 2 })).toBe('{"a":1,"b":2}'); + }); + + it('should handle null and undefined', () => { + expect(configTransformers.string(null)).toBeUndefined(); + expect(configTransformers.string(undefined)).toBeUndefined(); + }); + + it('should handle failed JSON stringification', () => { + const circular: any = {}; + + circular.self = circular; + + expect(configTransformers.string(circular)).toBeUndefined(); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts new file mode 100644 index 000000000..38a7bb5dd --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -0,0 +1,56 @@ +import { Transform } from 'class-transformer'; +import { + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, +} from 'class-validator'; + +import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; +import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; + +export function applyBasicValidators( + type: ConfigVariableType, + target: object, + propertyKey: string, + options?: ConfigVariableOptions, +): void { + switch (type) { + case 'boolean': + Transform(({ value }) => { + const result = configTransformers.boolean(value); + + return result !== undefined ? result : value; + })(target, propertyKey); + IsBoolean()(target, propertyKey); + break; + + case 'number': + Transform(({ value }) => { + const result = configTransformers.number(value); + + return result !== undefined ? result : value; + })(target, propertyKey); + IsNumber()(target, propertyKey); + break; + + case 'string': + IsString()(target, propertyKey); + break; + + case 'enum': + if (options) { + IsEnum(options)(target, propertyKey); + } + break; + + case 'array': + IsArray()(target, propertyKey); + break; + + default: + throw new Error(`Unsupported config variable type: ${type}`); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/config-transformers.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/config-transformers.util.ts new file mode 100644 index 000000000..5dbacf93c --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/config-transformers.util.ts @@ -0,0 +1,79 @@ +export const configTransformers = { + boolean: (value: unknown): boolean | undefined => { + if (value === null || value === undefined) { + return undefined; + } + + if (typeof value === 'boolean') { + return value; + } + + if (typeof value === 'number') { + return value !== 0; + } + + if (typeof value === 'string') { + const lowerValue = value.toLowerCase(); + + if (['true', 'on', 'yes', '1'].includes(lowerValue)) { + return true; + } + + if (['false', 'off', 'no', '0'].includes(lowerValue)) { + return false; + } + } + + return undefined; + }, + + number: (value: unknown): number | undefined => { + if (value === null || value === undefined) { + return undefined; + } + + if (typeof value === 'number') { + return value; + } + + if (typeof value === 'string') { + const parsedNumber = parseFloat(value); + + if (isNaN(parsedNumber)) { + return undefined; + } + + return parsedNumber; + } + + if (typeof value === 'boolean') { + return value ? 1 : 0; + } + + return undefined; + }, + + string: (value: unknown): string | undefined => { + if (value === null || value === undefined) { + return undefined; + } + + if (typeof value === 'string') { + return value; + } + + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + + if (Array.isArray(value) || typeof value === 'object') { + try { + return JSON.stringify(value); + } catch { + return undefined; + } + } + + return undefined; + }, +}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts new file mode 100644 index 000000000..ccd782d75 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts @@ -0,0 +1,10 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +export const isEnvOnlyConfigVar = (key: keyof ConfigVariables): boolean => { + const metadata = + TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; + const envMetadata = metadata[key]; + + return !!envMetadata?.isEnvOnly; +}; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts index 88bfa5a53..836586d57 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts @@ -28,7 +28,7 @@ xdescribe('Microsoft dev tests : get message list service', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessageListService, MicrosoftClientProvider, @@ -118,7 +118,7 @@ xdescribe('Microsoft dev tests : get full message list service for folders', () beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessageListService, MicrosoftClientProvider, @@ -207,7 +207,7 @@ xdescribe('Microsoft dev tests : get partial message list service for folders', beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessageListService, MicrosoftClientProvider, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts index 1010eb3f7..b9e577a0f 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts @@ -23,7 +23,7 @@ xdescribe('Microsoft dev tests : get messages service', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessagesService, MicrosoftHandleErrorService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts index afe372e39..5d7aba203 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts @@ -3,7 +3,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConnectedAccountProvider } from 'twenty-shared/types'; -import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; import { microsoftGraphBatchWithHtmlMessagesResponse, @@ -21,7 +21,6 @@ describe('Microsoft get messages service', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], providers: [ MicrosoftGetMessagesService, MicrosoftHandleErrorService, @@ -29,6 +28,10 @@ describe('Microsoft get messages service', () => { MicrosoftOAuth2ClientManagerService, MicrosoftFetchByBatchService, ConfigService, + { + provide: TwentyConfigService, + useValue: {}, + }, ], }).compile(); @@ -37,6 +40,10 @@ describe('Microsoft get messages service', () => { ); }); + afterEach(() => { + jest.clearAllMocks(); + }); + it('Should be defined', () => { expect(service).toBeDefined(); }); diff --git a/yarn.lock b/yarn.lock index 6e5a7dce3..36e6a027b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10561,6 +10561,20 @@ __metadata: languageName: node linkType: hard +"@nestjs/schedule@npm:^3.0.0": + version: 3.0.4 + resolution: "@nestjs/schedule@npm:3.0.4" + dependencies: + cron: "npm:2.4.3" + uuid: "npm:9.0.1" + peerDependencies: + "@nestjs/common": ^8.0.0 || ^9.0.0 || ^10.0.0 + "@nestjs/core": ^8.0.0 || ^9.0.0 || ^10.0.0 + reflect-metadata: ^0.1.12 + checksum: 10c0/10af832f611139b586bd85714da1697276ddf51513c91cb20c82eb481c9b1741ec85560f7c55d6e560c8ab7f4665f95cb9fd77241a84f2b432b03e2819ba0f86 + languageName: node + linkType: hard + "@nestjs/schematics@npm:^10.0.1": version: 10.1.3 resolution: "@nestjs/schematics@npm:10.1.3" @@ -22593,6 +22607,13 @@ __metadata: languageName: node linkType: hard +"@types/luxon@npm:~3.3.0": + version: 3.3.8 + resolution: "@types/luxon@npm:3.3.8" + checksum: 10c0/f2ffa31364eb94ca0474a196f533d301025a203bb2758ce0cf209f338cece0af169edea230b5c0b1a68a71adb02f369faa5ec0bd824deb8f0a08cac6803b1b06 + languageName: node + linkType: hard + "@types/markdown-it@npm:12.2.3": version: 12.2.3 resolution: "@types/markdown-it@npm:12.2.3" @@ -30201,6 +30222,16 @@ __metadata: languageName: node linkType: hard +"cron@npm:2.4.3": + version: 2.4.3 + resolution: "cron@npm:2.4.3" + dependencies: + "@types/luxon": "npm:~3.3.0" + luxon: "npm:~3.3.0" + checksum: 10c0/3112d4cb0aa1c1129c0bb742eec205e38948806c907e21a0680d1aa83a1270bfade9fcf090c1604529684d21a64d74eb89075e25c16d48d95cf3c5b5d032f316 + languageName: node + linkType: hard + "cross-env@npm:^7.0.3": version: 7.0.3 resolution: "cross-env@npm:7.0.3" @@ -42038,6 +42069,13 @@ __metadata: languageName: node linkType: hard +"luxon@npm:~3.3.0": + version: 3.3.0 + resolution: "luxon@npm:3.3.0" + checksum: 10c0/47f8e1e96b25441c799b8aa833b3f007fb1854713bcffc8c3384eda8e61fc9af1f038474d137274d2d386492f341c8a8c992fc78c213adfb3143780feba2776c + languageName: node + linkType: hard + "lz-string@npm:^1.4.4, lz-string@npm:^1.5.0": version: 1.5.0 resolution: "lz-string@npm:1.5.0" @@ -55047,6 +55085,7 @@ __metadata: "@nestjs/cli": "npm:10.3.0" "@nestjs/devtools-integration": "npm:^0.1.6" "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch" + "@nestjs/schedule": "npm:^3.0.0" "@node-saml/passport-saml": "npm:^5.0.0" "@nx/js": "npm:18.3.3" "@opentelemetry/api": "npm:^1.9.0"