Twenty config core implementation (#11595)

closes https://github.com/twentyhq/core-team-issues/issues/760

---------

Co-authored-by: Charles Bochet <charlesBochet@users.noreply.github.com>
Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
Co-authored-by: Félix Malfait <felix@twenty.com>
This commit is contained in:
nitin
2025-04-26 12:51:59 +05:30
committed by GitHub
parent bb8fa02899
commit a15b87649a
41 changed files with 3672 additions and 208 deletions

View File

@ -25,6 +25,7 @@
"@nestjs/cache-manager": "^2.2.1",
"@nestjs/devtools-integration": "^0.1.6",
"@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch",
"@nestjs/schedule": "^3.0.0",
"@node-saml/passport-saml": "^5.0.0",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/exporter-metrics-otlp-http": "^0.200.0",

View File

@ -20,6 +20,7 @@ const coreTypeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
@Module({
imports: [
TwentyConfigModule,
TypeOrmModule.forRootAsync({
useFactory: metadataTypeORMFactory,
name: 'metadata',
@ -28,7 +29,6 @@ const coreTypeORMFactory = async (): Promise<TypeOrmModuleOptions> => ({
useFactory: coreTypeORMFactory,
name: 'core',
}),
TwentyConfigModule,
],
providers: [TypeORMService],
exports: [TypeORMService],

View File

@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
import { HttpAdapterHost } from '@nestjs/core';
import { EventEmitterModule } from '@nestjs/event-emitter';
import { WorkspaceQueryRunnerModule } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.module';
import { ActorModule } from 'src/engine/core-modules/actor/actor.module';
import { AdminPanelModule } from 'src/engine/core-modules/admin-panel/admin-panel.module';
import { AppTokenModule } from 'src/engine/core-modules/app-token/app-token.module';
@ -47,9 +48,8 @@ import { WorkflowApiModule } from 'src/engine/core-modules/workflow/workflow-api
import { WorkspaceInvitationModule } from 'src/engine/core-modules/workspace-invitation/workspace-invitation.module';
import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module';
import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
import { WorkspaceQueryRunnerModule } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.module';
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
import { AnalyticsModule } from './analytics/analytics.module';
import { ClientConfigModule } from './client-config/client-config.module';
@ -57,6 +57,7 @@ import { FileModule } from './file/file.module';
@Module({
imports: [
TwentyConfigModule.forRoot(),
HealthModule,
AnalyticsModule,
AuthModule,
@ -81,7 +82,6 @@ import { FileModule } from './file/file.module';
AdminPanelModule,
LabModule,
RoleModule,
TwentyConfigModule,
RedisClientModule,
WorkspaceQueryRunnerModule,
SubscriptionsModule,

View File

@ -3,11 +3,10 @@ import { Module } from '@nestjs/common';
import { FileUploadResolver } from 'src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver';
import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service';
import { FileModule } from 'src/engine/core-modules/file/file.module';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
@Module({
imports: [FileModule],
providers: [FileUploadService, FileUploadResolver, TwentyConfigService],
providers: [FileUploadService, FileUploadResolver],
exports: [FileUploadService, FileUploadResolver],
})
export class FileUploadModule {}

View File

@ -6,7 +6,6 @@ import { FileWorkspaceFolderDeletionJob } from 'src/engine/core-modules/file/job
import { FileAttachmentListener } from 'src/engine/core-modules/file/listeners/file-attachment.listener';
import { FileWorkspaceMemberListener } from 'src/engine/core-modules/file/listeners/file-workspace-member.listener';
import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { FileController } from './controllers/file.controller';
import { FileService } from './services/file.service';
@ -15,7 +14,6 @@ import { FileService } from './services/file.service';
imports: [JwtModule],
providers: [
FileService,
TwentyConfigService,
FilePathGuard,
FileAttachmentListener,
FileWorkspaceMemberListener,

View File

@ -0,0 +1,196 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
describe('ConfigCacheService', () => {
let service: ConfigCacheService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ConfigCacheService],
}).compile();
service = module.get<ConfigCacheService>(ConfigCacheService);
});
afterEach(() => {
service.onModuleDestroy();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('get and set', () => {
it('should set and get a value from cache', () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const value = true;
service.set(key, value);
const result = service.get(key);
expect(result).toBe(value);
});
it('should return undefined for non-existent key', () => {
const result = service.get(
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
);
expect(result).toBeUndefined();
});
it('should handle different value types', () => {
const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables;
const numberKey = 'NODE_PORT' as keyof ConfigVariables;
service.set(booleanKey, true);
service.set(stringKey, 'test@example.com');
service.set(numberKey, 3000);
expect(service.get(booleanKey)).toBe(true);
expect(service.get(stringKey)).toBe('test@example.com');
expect(service.get(numberKey)).toBe(3000);
});
});
describe('negative lookup cache', () => {
it('should check if a negative cache entry exists', () => {
const key = 'TEST_KEY' as keyof ConfigVariables;
service.markKeyAsMissing(key);
const result = service.isKeyKnownMissing(key);
expect(result).toBe(true);
});
it('should return false for negative cache entry check when not in cache', () => {
const key = 'NON_EXISTENT_KEY' as keyof ConfigVariables;
const result = service.isKeyKnownMissing(key);
expect(result).toBe(false);
});
});
describe('clear operations', () => {
it('should clear specific key', () => {
const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables;
service.set(key1, true);
service.set(key2, 'test@example.com');
service.clear(key1);
expect(service.get(key1)).toBeUndefined();
expect(service.get(key2)).toBe('test@example.com');
});
it('should clear all entries', () => {
const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables;
service.set(key1, true);
service.set(key2, 'test@example.com');
service.clearAll();
expect(service.get(key1)).toBeUndefined();
expect(service.get(key2)).toBeUndefined();
});
});
describe('getCacheInfo', () => {
it('should return correct cache information', () => {
const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables;
const key3 = 'NODE_PORT' as keyof ConfigVariables;
service.set(key1, true);
service.set(key2, 'test@example.com');
service.markKeyAsMissing(key3);
const info = service.getCacheInfo();
expect(info.foundConfigValues).toBe(2);
expect(info.knownMissingKeys).toBe(1);
expect(info.cacheKeys).toContain(key1);
expect(info.cacheKeys).toContain(key2);
expect(info.cacheKeys).not.toContain(key3);
expect(service.isKeyKnownMissing(key3)).toBe(true);
});
it('should properly count cache entries', () => {
const key1 = 'KEY1' as keyof ConfigVariables;
const key2 = 'KEY2' as keyof ConfigVariables;
const key3 = 'KEY3' as keyof ConfigVariables;
// Add some values to the cache
service.set(key1, 'value1');
service.set(key2, 'value2');
service.markKeyAsMissing(key3);
const cacheInfo = service.getCacheInfo();
expect(cacheInfo.foundConfigValues).toBe(2);
expect(cacheInfo.knownMissingKeys).toBe(1);
expect(cacheInfo.cacheKeys).toContain(key1);
expect(cacheInfo.cacheKeys).toContain(key2);
expect(service.isKeyKnownMissing(key3)).toBe(true);
});
});
describe('module lifecycle', () => {
it('should clear cache on module destroy', () => {
const key = 'TEST_KEY' as keyof ConfigVariables;
service.set(key, 'test');
service.onModuleDestroy();
expect(service.get(key)).toBeUndefined();
});
});
describe('getAllKeys', () => {
it('should return all keys from both positive and negative caches', () => {
const positiveKey1 = 'POSITIVE_KEY1' as keyof ConfigVariables;
const positiveKey2 = 'POSITIVE_KEY2' as keyof ConfigVariables;
const negativeKey = 'NEGATIVE_KEY' as keyof ConfigVariables;
// Set up keys
service.set(positiveKey1, 'value1');
service.set(positiveKey2, 'value2');
service.markKeyAsMissing(negativeKey);
const allKeys = service.getAllKeys();
expect(allKeys).toContain(positiveKey1);
expect(allKeys).toContain(positiveKey2);
expect(allKeys).toContain(negativeKey);
});
it('should return empty array when no keys exist', () => {
const allKeys = service.getAllKeys();
expect(allKeys).toHaveLength(0);
});
it('should not have duplicates if a key somehow exists in both caches', () => {
const key = 'DUPLICATE_KEY' as keyof ConfigVariables;
// First add to positive cache
service.set(key, 'value');
// Then force it into negative cache (normally this would remove from positive)
// We're bypassing normal behavior for testing edge cases
service.addToMissingKeysForTesting(key);
const allKeys = service.getAllKeys();
// Should only appear once in the result
expect(allKeys.filter((k) => k === key)).toHaveLength(1);
});
});
});

View File

@ -0,0 +1,86 @@
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
import {
ConfigCacheEntry,
ConfigKey,
ConfigValue,
} from './interfaces/config-cache-entry.interface';
@Injectable()
export class ConfigCacheService implements OnModuleDestroy {
private readonly logger = new Logger(ConfigCacheService.name);
private readonly foundConfigValuesCache: Map<
ConfigKey,
ConfigCacheEntry<ConfigKey>
>;
private readonly knownMissingKeysCache: Set<ConfigKey>;
constructor() {
this.foundConfigValuesCache = new Map();
this.knownMissingKeysCache = new Set();
}
get<T extends ConfigKey>(key: T): ConfigValue<T> | undefined {
const entry = this.foundConfigValuesCache.get(key);
if (!entry) {
return undefined;
}
return entry.value as ConfigValue<T>;
}
isKeyKnownMissing(key: ConfigKey): boolean {
return this.knownMissingKeysCache.has(key);
}
set<T extends ConfigKey>(key: T, value: ConfigValue<T>): void {
this.foundConfigValuesCache.set(key, { value });
this.knownMissingKeysCache.delete(key);
}
markKeyAsMissing(key: ConfigKey): void {
this.knownMissingKeysCache.add(key);
this.foundConfigValuesCache.delete(key);
}
clear(key: ConfigKey): void {
this.foundConfigValuesCache.delete(key);
this.knownMissingKeysCache.delete(key);
}
clearAll(): void {
this.foundConfigValuesCache.clear();
this.knownMissingKeysCache.clear();
}
getCacheInfo(): {
foundConfigValues: number;
knownMissingKeys: number;
cacheKeys: string[];
} {
return {
foundConfigValues: this.foundConfigValuesCache.size,
knownMissingKeys: this.knownMissingKeysCache.size,
cacheKeys: Array.from(this.foundConfigValuesCache.keys()),
};
}
onModuleDestroy() {
this.clearAll();
}
getAllKeys(): ConfigKey[] {
const foundKeys = Array.from(this.foundConfigValuesCache.keys());
const missingKeys = Array.from(this.knownMissingKeysCache);
return [...new Set([...foundKeys, ...missingKeys])];
}
/**
* Helper method for testing edge cases
*/
addToMissingKeysForTesting(key: ConfigKey): void {
this.knownMissingKeysCache.add(key);
}
}

View File

@ -0,0 +1,8 @@
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
export type ConfigKey = keyof ConfigVariables;
export type ConfigValue<T extends ConfigKey> = ConfigVariables[T];
export interface ConfigCacheEntry<T extends ConfigKey> {
value: ConfigValue<T>;
}

View File

@ -0,0 +1,3 @@
export const CONFIG_VARIABLES_INSTANCE_TOKEN = Symbol(
'CONFIG_VARIABLES_INSTANCE_TOKEN',
);

View File

@ -0,0 +1 @@
export const CONFIG_VARIABLES_REFRESH_CRON_INTERVAL = '*/15 * * * * *';

View File

@ -0,0 +1,472 @@
import { LogLevel } from '@nestjs/common';
import { Test, TestingModule } from '@nestjs/testing';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants';
import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service';
import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum';
import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util';
import { TypedReflect } from 'src/utils/typed-reflect';
// Mock configTransformers for type validation tests
jest.mock(
'src/engine/core-modules/twenty-config/utils/config-transformers.util',
() => {
const originalModule = jest.requireActual(
'src/engine/core-modules/twenty-config/utils/config-transformers.util',
);
return {
configTransformers: {
...originalModule.configTransformers,
// These mocked versions can be overridden in specific tests
_mockedBoolean: jest.fn(),
_mockedNumber: jest.fn(),
_mockedString: jest.fn(),
},
};
},
);
describe('ConfigValueConverterService', () => {
let service: ConfigValueConverterService;
beforeEach(async () => {
const mockConfigVariables = {
NODE_PORT: 3000,
};
const module: TestingModule = await Test.createTestingModule({
providers: [
ConfigValueConverterService,
{
provide: CONFIG_VARIABLES_INSTANCE_TOKEN,
useValue: mockConfigVariables,
},
],
}).compile();
service = module.get<ConfigValueConverterService>(
ConfigValueConverterService,
);
});
describe('convertDbValueToAppValue', () => {
it('should convert string to boolean based on metadata', () => {
// Mock the metadata
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
AUTH_PASSWORD_ENABLED: {
type: 'boolean',
group: ConfigVariablesGroup.Other,
description: 'Enable or disable password authentication for users',
},
});
expect(
service.convertDbValueToAppValue(
'true',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(true);
expect(
service.convertDbValueToAppValue(
'True',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(true);
expect(
service.convertDbValueToAppValue(
'yes',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(true);
expect(
service.convertDbValueToAppValue(
'1',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(true);
expect(
service.convertDbValueToAppValue(
'false',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(false);
expect(
service.convertDbValueToAppValue(
'False',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(false);
expect(
service.convertDbValueToAppValue(
'no',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(false);
expect(
service.convertDbValueToAppValue(
'0',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(false);
});
it('should convert string to number based on metadata', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
NODE_PORT: {
type: 'number',
group: ConfigVariablesGroup.ServerConfig,
description: 'Port for the node server',
},
});
expect(
service.convertDbValueToAppValue(
'42',
'NODE_PORT' as keyof ConfigVariables,
),
).toBe(42);
expect(
service.convertDbValueToAppValue(
'3.14',
'NODE_PORT' as keyof ConfigVariables,
),
).toBe(3.14);
expect(
service.convertDbValueToAppValue(
'not-a-number',
'NODE_PORT' as keyof ConfigVariables,
),
).toBeUndefined();
});
it('should convert string to array based on metadata', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVELS: {
type: 'array',
group: ConfigVariablesGroup.Logging,
description: 'Levels of logging to be captured',
},
});
expect(
service.convertDbValueToAppValue(
'log,error,warn',
'LOG_LEVELS' as keyof ConfigVariables,
),
).toEqual(['log', 'error', 'warn']);
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVELS: {
type: 'array',
group: ConfigVariablesGroup.Logging,
description: 'Levels of logging to be captured',
},
});
expect(
service.convertDbValueToAppValue(
'["log","error","warn"]',
'LOG_LEVELS' as keyof ConfigVariables,
),
).toEqual(['log', 'error', 'warn']);
});
it('should handle enum values as strings', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined);
expect(
service.convertDbValueToAppValue(
'development',
'NODE_ENV' as keyof ConfigVariables,
),
).toBe('development');
});
it('should handle various input types', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
AUTH_PASSWORD_ENABLED: {
type: 'boolean',
group: ConfigVariablesGroup.Other,
description: 'Enable or disable password authentication for users',
},
});
expect(
service.convertDbValueToAppValue(
true,
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
),
).toBe(true);
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
NODE_PORT: {
type: 'number',
group: ConfigVariablesGroup.ServerConfig,
description: 'Port for the node server',
},
});
expect(
service.convertDbValueToAppValue(
42,
'NODE_PORT' as keyof ConfigVariables,
),
).toBe(42);
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVELS: {
type: 'array',
group: ConfigVariablesGroup.Logging,
description: 'Levels of logging to be captured',
},
});
expect(
service.convertDbValueToAppValue(
['log', 'error'] as LogLevel[],
'LOG_LEVELS' as keyof ConfigVariables,
),
).toEqual(['log', 'error']);
});
it('should fall back to default value approach when no metadata', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined);
expect(
service.convertDbValueToAppValue(
'42',
'NODE_PORT' as keyof ConfigVariables,
),
).toBe(42);
});
it('should handle null and undefined values', () => {
expect(
service.convertDbValueToAppValue(
null,
'NODE_PORT' as keyof ConfigVariables,
),
).toBeUndefined();
expect(
service.convertDbValueToAppValue(
undefined,
'NODE_PORT' as keyof ConfigVariables,
),
).toBeUndefined();
});
it('should throw error if boolean converter returns non-boolean', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
AUTH_PASSWORD_ENABLED: {
type: 'boolean',
group: ConfigVariablesGroup.Other,
description: 'Test boolean',
},
});
const originalBoolean = configTransformers.boolean;
configTransformers.boolean = jest
.fn()
.mockImplementation(() => 'not-a-boolean');
expect(() => {
service.convertDbValueToAppValue(
'true',
'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables,
);
}).toThrow(/Expected boolean for key AUTH_PASSWORD_ENABLED/);
configTransformers.boolean = originalBoolean;
});
it('should throw error if number converter returns non-number', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
NODE_PORT: {
type: 'number',
group: ConfigVariablesGroup.ServerConfig,
description: 'Test number',
},
});
const originalNumber = configTransformers.number;
configTransformers.number = jest
.fn()
.mockImplementation(() => 'not-a-number');
expect(() => {
service.convertDbValueToAppValue(
'42',
'NODE_PORT' as keyof ConfigVariables,
);
}).toThrow(/Expected number for key NODE_PORT/);
configTransformers.number = originalNumber;
});
it('should throw error if string converter returns non-string', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
EMAIL_FROM_ADDRESS: {
type: 'string',
group: ConfigVariablesGroup.EmailSettings,
description: 'Test string',
},
});
const originalString = configTransformers.string;
configTransformers.string = jest.fn().mockImplementation(() => 42);
expect(() => {
service.convertDbValueToAppValue(
'test@example.com',
'EMAIL_FROM_ADDRESS' as keyof ConfigVariables,
);
}).toThrow(/Expected string for key EMAIL_FROM_ADDRESS/);
configTransformers.string = originalString;
});
it('should throw error if array conversion produces non-array', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVELS: {
type: 'array',
group: ConfigVariablesGroup.Logging,
description: 'Test array',
},
});
const convertToArraySpy = jest
.spyOn(
service as any, // Cast to any to access private method
'convertToArray',
)
.mockReturnValueOnce('not-an-array');
expect(() => {
service.convertDbValueToAppValue(
'log,error,warn',
'LOG_LEVELS' as keyof ConfigVariables,
);
}).toThrow(/Expected array for key LOG_LEVELS/);
convertToArraySpy.mockRestore();
});
it('should handle array with option validation', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVELS: {
type: 'array',
group: ConfigVariablesGroup.Logging,
description: 'Test array with options',
options: ['log', 'error', 'warn', 'debug'],
},
});
expect(
service.convertDbValueToAppValue(
'log,error,warn',
'LOG_LEVELS' as keyof ConfigVariables,
),
).toEqual(['log', 'error', 'warn']);
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVELS: {
type: 'array',
group: ConfigVariablesGroup.Logging,
description: 'Test array with options',
options: ['log', 'error', 'warn', 'debug'],
},
});
expect(
service.convertDbValueToAppValue(
'log,invalid,warn',
'LOG_LEVELS' as keyof ConfigVariables,
),
).toEqual(['log', 'warn']);
});
it('should properly handle enum with options', () => {
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVEL: {
type: 'enum',
group: ConfigVariablesGroup.Logging,
description: 'Test enum',
options: ['log', 'error', 'warn', 'debug'],
},
});
expect(
service.convertDbValueToAppValue(
'error',
'LOG_LEVEL' as keyof ConfigVariables,
),
).toBe('error');
jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({
LOG_LEVEL: {
type: 'enum',
group: ConfigVariablesGroup.Logging,
description: 'Test enum',
options: ['log', 'error', 'warn', 'debug'],
},
});
expect(
service.convertDbValueToAppValue(
'invalid',
'LOG_LEVEL' as keyof ConfigVariables,
),
).toBeUndefined();
});
});
describe('convertAppValueToDbValue', () => {
it('should handle primitive types directly', () => {
expect(service.convertAppValueToDbValue('string-value' as any)).toBe(
'string-value',
);
expect(service.convertAppValueToDbValue(42 as any)).toBe(42);
expect(service.convertAppValueToDbValue(true as any)).toBe(true);
expect(service.convertAppValueToDbValue(undefined as any)).toBe(null);
});
it('should handle arrays', () => {
const array = ['log', 'error', 'warn'] as LogLevel[];
expect(service.convertAppValueToDbValue(array as any)).toEqual(array);
});
it('should handle objects', () => {
const obj = { key: 'value' };
expect(service.convertAppValueToDbValue(obj as any)).toEqual(obj);
});
it('should convert null to null', () => {
expect(service.convertAppValueToDbValue(null as any)).toBe(null);
});
it('should throw error for unsupported types', () => {
const symbol = Symbol('test');
expect(() => {
service.convertAppValueToDbValue(symbol as any);
}).toThrow(/Cannot convert value of type symbol/);
});
it('should handle serialization errors', () => {
// Create an object with circular reference
const circular: any = {};
circular.self = circular;
expect(() => {
service.convertAppValueToDbValue(circular as any);
}).toThrow(/Failed to serialize object value/);
});
});
});

View File

@ -0,0 +1,213 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants';
import { ConfigVariablesMetadataMap } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator';
import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type';
import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type';
import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util';
import { TypedReflect } from 'src/utils/typed-reflect';
@Injectable()
export class ConfigValueConverterService {
private readonly logger = new Logger(ConfigValueConverterService.name);
constructor(
@Inject(CONFIG_VARIABLES_INSTANCE_TOKEN)
private readonly configVariables: ConfigVariables,
) {}
convertDbValueToAppValue<T extends keyof ConfigVariables>(
dbValue: unknown,
key: T,
): ConfigVariables[T] | undefined {
if (dbValue === null || dbValue === undefined) {
return undefined;
}
const metadata = this.getConfigVariableMetadata(key);
const configType = metadata?.type || this.inferTypeFromValue(key);
const options = metadata?.options;
try {
switch (configType) {
case 'boolean': {
const result = configTransformers.boolean(dbValue);
if (result !== undefined && typeof result !== 'boolean') {
throw new Error(
`Expected boolean for key ${key}, got ${typeof result}`,
);
}
return result as ConfigVariables[T];
}
case 'number': {
const result = configTransformers.number(dbValue);
if (result !== undefined && typeof result !== 'number') {
throw new Error(
`Expected number for key ${key}, got ${typeof result}`,
);
}
return result as ConfigVariables[T];
}
case 'string': {
const result = configTransformers.string(dbValue);
if (result !== undefined && typeof result !== 'string') {
throw new Error(
`Expected string for key ${key}, got ${typeof result}`,
);
}
return result as ConfigVariables[T];
}
case 'array': {
const result = this.convertToArray(dbValue, options);
if (result !== undefined && !Array.isArray(result)) {
throw new Error(
`Expected array for key ${key}, got ${typeof result}`,
);
}
return result as ConfigVariables[T];
}
case 'enum': {
const result = this.convertToEnum(dbValue, options);
return result as ConfigVariables[T];
}
default:
return dbValue as ConfigVariables[T];
}
} catch (error) {
throw new Error(
`Failed to convert ${key as string} to app value: ${(error as Error).message}`,
);
}
}
convertAppValueToDbValue<T extends keyof ConfigVariables>(
appValue: ConfigVariables[T] | null | undefined,
): unknown {
if (appValue === undefined || appValue === null) {
return null;
}
if (
typeof appValue === 'string' ||
typeof appValue === 'number' ||
typeof appValue === 'boolean'
) {
return appValue;
}
if (Array.isArray(appValue)) {
return appValue;
}
if (typeof appValue === 'object') {
try {
return JSON.parse(JSON.stringify(appValue));
} catch (error) {
throw new Error(
`Failed to serialize object value: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
throw new Error(
`Cannot convert value of type ${typeof appValue} to storage format`,
);
}
private convertToArray(
value: unknown,
options?: ConfigVariableOptions,
): unknown[] {
if (Array.isArray(value)) {
return this.validateArrayAgainstOptions(value, options);
}
if (typeof value === 'string') {
try {
const parsedArray = JSON.parse(value);
if (Array.isArray(parsedArray)) {
return this.validateArrayAgainstOptions(parsedArray, options);
}
} catch {
const splitArray = value.split(',').map((item) => item.trim());
return this.validateArrayAgainstOptions(splitArray, options);
}
}
return this.validateArrayAgainstOptions([value], options);
}
private validateArrayAgainstOptions(
array: unknown[],
options?: ConfigVariableOptions,
): unknown[] {
if (!options || !Array.isArray(options) || options.length === 0) {
return array;
}
return array.filter((item) => {
const included = options.includes(item as string);
if (!included) {
this.logger.debug(
`Filtered out array item '${String(item)}' not in allowed options`,
);
}
return included;
});
}
private convertToEnum(
value: unknown,
options?: ConfigVariableOptions,
): unknown | undefined {
if (!options || !Array.isArray(options) || options.length === 0) {
return value;
}
if (options.includes(value as string)) {
return value;
}
return undefined;
}
private getConfigVariableMetadata<T extends keyof ConfigVariables>(key: T) {
const allMetadata = TypedReflect.getMetadata(
'config-variables',
ConfigVariables.prototype.constructor,
) as ConfigVariablesMetadataMap | undefined;
return allMetadata?.[key as string];
}
private inferTypeFromValue<T extends keyof ConfigVariables>(
key: T,
): ConfigVariableType {
const defaultValue = this.configVariables[key];
if (typeof defaultValue === 'boolean') return 'boolean';
if (typeof defaultValue === 'number') return 'number';
if (Array.isArray(defaultValue)) return 'array';
return 'string';
}
}

View File

@ -1,18 +0,0 @@
import { Transform } from 'class-transformer';
export const CastToBoolean = () =>
Transform(({ value }: { value: string }) => toBoolean(value));
const toBoolean = (value: any) => {
if (typeof value === 'boolean') {
return value;
}
if (['true', 'on', 'yes', '1'].includes(value.toLowerCase())) {
return true;
}
if (['false', 'off', 'no', '0'].includes(value.toLowerCase())) {
return false;
}
return undefined;
};

View File

@ -1,12 +0,0 @@
import { Transform } from 'class-transformer';
export const CastToStringArray = () =>
Transform(({ value }: { value: string }) => toStringArray(value));
const toStringArray = (value: any) => {
if (typeof value === 'string') {
return value.split(',').map((item) => item.trim());
}
return undefined;
};

View File

@ -1,12 +1,22 @@
import { registerDecorator, ValidationOptions } from 'class-validator';
import {
IsOptional,
registerDecorator,
ValidationOptions,
} from 'class-validator';
import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum';
import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type';
import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type';
import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util';
import { TypedReflect } from 'src/utils/typed-reflect';
export interface ConfigVariablesMetadataOptions {
group: ConfigVariablesGroup;
description: string;
isSensitive?: boolean;
isEnvOnly?: boolean;
type?: ConfigVariableType;
options?: ConfigVariableOptions;
}
export type ConfigVariablesMetadataMap = {
@ -30,6 +40,26 @@ export function ConfigVariablesMetadata(
target.constructor,
);
const propertyDescriptor = Object.getOwnPropertyDescriptor(
target.constructor.prototype,
propertyKey,
);
const hasDefaultValue =
propertyDescriptor && propertyDescriptor.value !== undefined;
if (!hasDefaultValue) {
IsOptional()(target, propertyKey);
}
if (options.type) {
applyBasicValidators(
options.type,
target,
propertyKey.toString(),
options.options,
);
}
registerDecorator({
name: propertyKey.toString(),
target: target.constructor,

View File

@ -0,0 +1,374 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver';
import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service';
import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util';
jest.mock(
'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util',
() => ({
isEnvOnlyConfigVar: jest.fn(),
}),
);
const CONFIG_PASSWORD_KEY = 'AUTH_PASSWORD_ENABLED';
const CONFIG_EMAIL_KEY = 'EMAIL_FROM_ADDRESS';
const CONFIG_ENV_ONLY_KEY = 'ENV_ONLY_VAR';
const CONFIG_PORT_KEY = 'NODE_PORT';
class TestDatabaseConfigDriver extends DatabaseConfigDriver {
// Expose the protected/private property for testing
public get testAllPossibleConfigKeys(): Array<keyof ConfigVariables> {
return this['allPossibleConfigKeys'];
}
// Override Object.keys usage in constructor with our test keys
constructor(
configCache: ConfigCacheService,
configStorage: ConfigStorageService,
) {
super(configCache, configStorage);
Object.defineProperty(this, 'allPossibleConfigKeys', {
value: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY, CONFIG_PORT_KEY],
writable: false,
configurable: true,
});
}
}
describe('DatabaseConfigDriver', () => {
let driver: TestDatabaseConfigDriver;
let configCache: ConfigCacheService;
let configStorage: ConfigStorageService;
beforeEach(async () => {
(isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => {
return key === CONFIG_ENV_ONLY_KEY;
});
const module: TestingModule = await Test.createTestingModule({
providers: [
{
provide: DatabaseConfigDriver,
useClass: TestDatabaseConfigDriver,
},
{
provide: ConfigCacheService,
useValue: {
get: jest.fn(),
set: jest.fn(),
clear: jest.fn(),
clearAll: jest.fn(),
isKeyKnownMissing: jest.fn(),
markKeyAsMissing: jest.fn(),
getCacheInfo: jest.fn(),
getAllKeys: jest.fn(),
},
},
{
provide: ConfigStorageService,
useValue: {
get: jest.fn(),
set: jest.fn(),
loadAll: jest.fn(),
},
},
],
}).compile();
driver = module.get<TestDatabaseConfigDriver>(
DatabaseConfigDriver,
) as TestDatabaseConfigDriver;
configCache = module.get<ConfigCacheService>(ConfigCacheService);
configStorage = module.get<ConfigStorageService>(ConfigStorageService);
jest.clearAllMocks();
});
afterEach(() => {
jest.clearAllMocks();
jest.useRealTimers();
});
it('should be defined', () => {
expect(driver).toBeDefined();
});
describe('initialization', () => {
it('should have allPossibleConfigKeys properly set', () => {
expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_PASSWORD_KEY);
expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_EMAIL_KEY);
expect(driver.testAllPossibleConfigKeys).not.toContain(
CONFIG_ENV_ONLY_KEY,
);
});
it('should initialize successfully with DB values and mark missing keys', async () => {
const configVars = new Map();
configVars.set(CONFIG_PASSWORD_KEY, true);
jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars);
await driver.onModuleInit();
expect(configStorage.loadAll).toHaveBeenCalled();
expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true);
expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(
CONFIG_EMAIL_KEY,
);
expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith(
CONFIG_ENV_ONLY_KEY,
);
});
it('should handle initialization failure gracefully', async () => {
const error = new Error('DB error');
jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error);
jest.spyOn(driver['logger'], 'error').mockImplementation();
// Should not throw because we're handling errors internally now
await driver.onModuleInit();
expect(driver['logger'].error).toHaveBeenCalled();
expect(configStorage.loadAll).toHaveBeenCalled();
});
});
describe('get', () => {
it('should return cached value when available', async () => {
const cachedValue = true;
jest.spyOn(configCache, 'get').mockReturnValue(cachedValue);
const result = driver.get(CONFIG_PASSWORD_KEY);
expect(result).toBe(cachedValue);
expect(configCache.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY);
});
it('should return undefined when value is not in cache', async () => {
jest.spyOn(configCache, 'get').mockReturnValue(undefined);
const result = driver.get(CONFIG_PASSWORD_KEY);
expect(result).toBeUndefined();
expect(configCache.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY);
});
it('should handle different config variable types correctly', () => {
const stringValue = 'test@example.com';
const booleanValue = true;
const numberValue = 3000;
jest.spyOn(configCache, 'get').mockImplementation((key) => {
switch (key) {
case CONFIG_EMAIL_KEY:
return stringValue;
case CONFIG_PASSWORD_KEY:
return booleanValue;
case CONFIG_PORT_KEY:
return numberValue;
default:
return undefined;
}
});
expect(driver.get(CONFIG_EMAIL_KEY)).toBe(stringValue);
expect(driver.get(CONFIG_PASSWORD_KEY)).toBe(booleanValue);
expect(driver.get(CONFIG_PORT_KEY)).toBe(numberValue);
});
});
describe('update', () => {
beforeEach(async () => {
(isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false);
});
it('should update config in storage and cache', async () => {
const value = true;
await driver.update(CONFIG_PASSWORD_KEY, value);
expect(configStorage.set).toHaveBeenCalledWith(
CONFIG_PASSWORD_KEY,
value,
);
expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value);
});
it('should throw error when updating env-only variable', async () => {
const value = true;
(isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true);
await expect(driver.update(CONFIG_PASSWORD_KEY, value)).rejects.toThrow();
});
});
describe('fetchAndCacheConfigVariable', () => {
it('should refresh config variable from storage', async () => {
const value = true;
jest.spyOn(configStorage, 'get').mockResolvedValue(value);
await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY);
expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY);
expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value);
});
it('should mark key as missing when value is undefined', async () => {
jest.spyOn(configStorage, 'get').mockResolvedValue(undefined);
await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY);
expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY);
expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(
CONFIG_PASSWORD_KEY,
);
expect(configCache.set).not.toHaveBeenCalled();
});
it('should mark key as missing when storage fetch fails', async () => {
const error = new Error('Storage error');
jest.spyOn(configStorage, 'get').mockRejectedValue(error);
const loggerSpy = jest
.spyOn(driver['logger'], 'error')
.mockImplementation();
await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY);
expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY);
expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(
CONFIG_PASSWORD_KEY,
);
expect(loggerSpy).toHaveBeenCalledWith(
expect.stringContaining('Failed to fetch config'),
error,
);
});
});
describe('cache operations', () => {
it('should return cache info', () => {
const cacheInfo = {
foundConfigValues: 2,
knownMissingKeys: 1,
cacheKeys: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY],
};
jest.spyOn(configCache, 'getCacheInfo').mockReturnValue(cacheInfo);
const result = driver.getCacheInfo();
expect(result).toEqual(cacheInfo);
});
});
describe('refreshAllCache', () => {
it('should load all config values from DB', async () => {
const dbValues = new Map();
dbValues.set(CONFIG_PASSWORD_KEY, true);
dbValues.set(CONFIG_EMAIL_KEY, 'test@example.com');
jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues);
await driver.refreshAllCache();
expect(configStorage.loadAll).toHaveBeenCalled();
expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true);
expect(configCache.set).toHaveBeenCalledWith(
CONFIG_EMAIL_KEY,
'test@example.com',
);
});
it('should not affect env-only variables when found in DB', async () => {
const dbValues = new Map();
dbValues.set(CONFIG_PASSWORD_KEY, true);
dbValues.set(CONFIG_ENV_ONLY_KEY, 'env-value');
jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues);
await driver.refreshAllCache();
expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true);
expect(configCache.set).not.toHaveBeenCalledWith(
CONFIG_ENV_ONLY_KEY,
'env-value',
);
});
it('should mark keys as missing when not found in DB', async () => {
jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map());
await driver.refreshAllCache();
expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(
CONFIG_PASSWORD_KEY,
);
expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(
CONFIG_EMAIL_KEY,
);
expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith(
CONFIG_ENV_ONLY_KEY,
);
});
it('should properly handle mix of found and missing keys', async () => {
const dbValues = new Map();
dbValues.set(CONFIG_PASSWORD_KEY, true);
jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues);
await driver.refreshAllCache();
expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true);
expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(
CONFIG_EMAIL_KEY,
);
});
it('should handle errors gracefully and verify cache remains unchanged', async () => {
const error = new Error('Database error');
jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error);
jest.spyOn(driver['logger'], 'error').mockImplementation();
const mockCacheState = new Map();
mockCacheState.set(CONFIG_PASSWORD_KEY, false);
jest
.spyOn(configCache, 'getAllKeys')
.mockReturnValue([CONFIG_PASSWORD_KEY]);
jest
.spyOn(configCache, 'get')
.mockImplementation((key) => mockCacheState.get(key));
await driver.refreshAllCache();
expect(driver['logger'].error).toHaveBeenCalled();
expect(configStorage.loadAll).toHaveBeenCalled();
expect(configCache.set).not.toHaveBeenCalled();
expect(configCache.markKeyAsMissing).not.toHaveBeenCalled();
expect(configCache.clear).not.toHaveBeenCalled();
expect(configCache.clearAll).not.toHaveBeenCalled();
});
});
});

View File

@ -0,0 +1,110 @@
import { ConfigService } from '@nestjs/config';
import { Test, TestingModule } from '@nestjs/testing';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants';
import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver';
describe('EnvironmentConfigDriver', () => {
let driver: EnvironmentConfigDriver;
let configService: ConfigService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
EnvironmentConfigDriver,
{
provide: ConfigService,
useValue: {
get: jest.fn(),
},
},
{
provide: CONFIG_VARIABLES_INSTANCE_TOKEN,
useValue: new ConfigVariables(),
},
],
}).compile();
driver = module.get<EnvironmentConfigDriver>(EnvironmentConfigDriver);
configService = module.get<ConfigService>(ConfigService);
});
afterEach(() => {
jest.clearAllMocks();
});
it('should be defined', () => {
expect(driver).toBeDefined();
});
describe('get', () => {
it('should return value from config service when available', () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const expectedValue = true;
const defaultValue = new ConfigVariables()[key];
jest.spyOn(configService, 'get').mockReturnValue(expectedValue);
const result = driver.get(key);
expect(result).toBe(expectedValue);
expect(configService.get).toHaveBeenCalledWith(key, defaultValue);
});
it('should return default value when config service returns undefined', () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const defaultValue = new ConfigVariables()[key];
jest
.spyOn(configService, 'get')
.mockImplementation((_, defaultVal) => defaultVal);
const result = driver.get(key);
expect(result).toBe(defaultValue);
expect(configService.get).toHaveBeenCalledWith(key, defaultValue);
});
it('should handle different config variable types', () => {
const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables;
const numberKey = 'NODE_PORT' as keyof ConfigVariables;
const defaultValues = new ConfigVariables();
jest
.spyOn(configService, 'get')
.mockImplementation((key: keyof ConfigVariables) => {
switch (key) {
case booleanKey:
return true;
case stringKey:
return 'test@example.com';
case numberKey:
return 3000;
default:
return undefined;
}
});
expect(driver.get(booleanKey)).toBe(true);
expect(configService.get).toHaveBeenCalledWith(
booleanKey,
defaultValues[booleanKey],
);
expect(driver.get(stringKey)).toBe('test@example.com');
expect(configService.get).toHaveBeenCalledWith(
stringKey,
defaultValues[stringKey],
);
expect(driver.get(numberKey)).toBe(3000);
expect(configService.get).toHaveBeenCalledWith(
numberKey,
defaultValues[numberKey],
);
});
});
});

View File

@ -0,0 +1,193 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface';
import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_REFRESH_CRON_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants';
import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service';
import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util';
@Injectable()
export class DatabaseConfigDriver
implements DatabaseConfigDriverInterface, OnModuleInit
{
private readonly logger = new Logger(DatabaseConfigDriver.name);
private readonly allPossibleConfigKeys: Array<keyof ConfigVariables>;
constructor(
private readonly configCache: ConfigCacheService,
private readonly configStorage: ConfigStorageService,
) {
const allKeys = Object.keys(new ConfigVariables()) as Array<
keyof ConfigVariables
>;
this.allPossibleConfigKeys = allKeys.filter(
(key) => !isEnvOnlyConfigVar(key),
);
this.logger.debug(
'[INIT] Database config driver created, monitoring keys: ' +
this.allPossibleConfigKeys.length,
);
}
async onModuleInit(): Promise<void> {
try {
this.logger.log('[INIT] Loading initial config variables from database');
const loadedCount = await this.loadAllConfigVarsFromDb();
this.logger.log(
`[INIT] Config variables loaded: ${loadedCount} values found, ${this.allPossibleConfigKeys.length - loadedCount} missing`,
);
} catch (error) {
this.logger.error(
'[INIT] Failed to load config variables from database, falling back to environment variables',
error instanceof Error ? error.stack : error,
);
// Don't rethrow to allow the application to continue
// The driver's cache will be empty but the service will fall back to env vars
}
}
get<T extends keyof ConfigVariables>(key: T): ConfigVariables[T] | undefined {
return this.configCache.get(key);
}
async update<T extends keyof ConfigVariables>(
key: T,
value: ConfigVariables[T],
): Promise<void> {
if (isEnvOnlyConfigVar(key)) {
throw new Error(
`Cannot update environment-only variable: ${key as string}`,
);
}
try {
await this.configStorage.set(key, value);
this.configCache.set(key, value);
this.logger.debug(
`[UPDATE] Config variable ${key as string} updated successfully`,
);
} catch (error) {
this.logger.error(
`[UPDATE] Failed to update config variable ${key as string}`,
error,
);
throw error;
}
}
async fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise<void> {
try {
const value = await this.configStorage.get(key);
if (value !== undefined) {
this.configCache.set(key, value);
this.logger.debug(
`[FETCH] Config variable ${key as string} loaded from database`,
);
} else {
this.configCache.markKeyAsMissing(key);
this.logger.debug(
`[FETCH] Config variable ${key as string} not found in database, marked as missing`,
);
}
} catch (error) {
this.logger.error(
`[FETCH] Failed to fetch config variable ${key as string} from database`,
error,
);
this.configCache.markKeyAsMissing(key);
}
}
getCacheInfo(): {
foundConfigValues: number;
knownMissingKeys: number;
cacheKeys: string[];
} {
return this.configCache.getCacheInfo();
}
private async loadAllConfigVarsFromDb(): Promise<number> {
try {
this.logger.debug('[LOAD] Fetching all config variables from database');
const configVars = await this.configStorage.loadAll();
this.logger.debug(
`[LOAD] Processing ${this.allPossibleConfigKeys.length} possible config variables`,
);
for (const [key, value] of configVars.entries()) {
this.configCache.set(key, value);
}
for (const key of this.allPossibleConfigKeys) {
if (!configVars.has(key)) {
this.configCache.markKeyAsMissing(key);
}
}
const missingKeysCount =
this.allPossibleConfigKeys.length - configVars.size;
this.logger.debug(
`[LOAD] Cached ${configVars.size} config variables, marked ${missingKeysCount} keys as missing`,
);
return configVars.size;
} catch (error) {
this.logger.error(
'[LOAD] Failed to load config variables from database',
error,
);
throw error;
}
}
/**
* Refreshes all database-backed config variables.
* This method runs on a schedule and fetches all configs in one database query,
* then updates the cache with fresh values.
*/
@Cron(CONFIG_VARIABLES_REFRESH_CRON_INTERVAL)
async refreshAllCache(): Promise<void> {
try {
this.logger.debug(
'[REFRESH] Starting scheduled refresh of config variables',
);
const dbValues = await this.configStorage.loadAll();
this.logger.debug(
`[REFRESH] Processing ${this.allPossibleConfigKeys.length} possible config variables`,
);
for (const [key, value] of dbValues.entries()) {
if (!isEnvOnlyConfigVar(key)) {
this.configCache.set(key, value);
}
}
for (const key of this.allPossibleConfigKeys) {
if (!dbValues.has(key)) {
this.configCache.markKeyAsMissing(key);
}
}
const missingKeysCount =
this.allPossibleConfigKeys.length - dbValues.size;
this.logger.log(
`[REFRESH] Config variables refreshed: ${dbValues.size} values updated, ${missingKeysCount} marked as missing`,
);
} catch (error) {
this.logger.error('[REFRESH] Failed to refresh config variables', error);
// Error is caught and logged but not rethrown to prevent the cron job from crashing
}
}
}

View File

@ -0,0 +1,35 @@
import { DynamicModule, Module } from '@nestjs/common';
import { ScheduleModule } from '@nestjs/schedule';
import { TypeOrmModule } from '@nestjs/typeorm';
import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity';
import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants';
import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service';
import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver';
import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service';
@Module({})
export class DatabaseConfigModule {
static forRoot(): DynamicModule {
return {
module: DatabaseConfigModule,
imports: [
TypeOrmModule.forFeature([KeyValuePair], 'core'),
ScheduleModule.forRoot(),
],
providers: [
DatabaseConfigDriver,
ConfigCacheService,
ConfigStorageService,
ConfigValueConverterService,
{
provide: CONFIG_VARIABLES_INSTANCE_TOKEN,
useValue: new ConfigVariables(),
},
],
exports: [DatabaseConfigDriver],
};
}
}

View File

@ -0,0 +1,21 @@
import { Inject, Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants';
@Injectable()
export class EnvironmentConfigDriver {
constructor(
private readonly configService: ConfigService,
@Inject(CONFIG_VARIABLES_INSTANCE_TOKEN)
private readonly defaultConfigVariables: ConfigVariables,
) {}
get<T extends keyof ConfigVariables>(key: T): ConfigVariables[T] {
return this.configService.get<ConfigVariables[T]>(
key,
this.defaultConfigVariables[key],
);
}
}

View File

@ -0,0 +1,40 @@
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
/**
* Interface for drivers that support database-backed configuration
* with caching capabilities
*/
export interface DatabaseConfigDriverInterface {
/**
* Get a configuration value from cache
* Returns undefined if not in cache
*/
get<T extends keyof ConfigVariables>(key: T): ConfigVariables[T] | undefined;
/**
* Update a configuration value in the database and cache
*/
update<T extends keyof ConfigVariables>(
key: T,
value: ConfigVariables[T],
): Promise<void>;
/**
* Fetch and cache a specific configuration from its source
*/
fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise<void>;
/**
* Refreshes all entries in the config cache
*/
refreshAllCache(): Promise<void>;
/**
* Get information about the cache state
*/
getCacheInfo(): {
foundConfigValues: number;
knownMissingKeys: number;
cacheKeys: string[];
};
}

View File

@ -0,0 +1,5 @@
export enum ConfigSource {
ENVIRONMENT = 'ENVIRONMENT',
DATABASE = 'DATABASE',
DEFAULT = 'DEFAULT',
}

View File

@ -0,0 +1,488 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { DeleteResult, IsNull, Repository } from 'typeorm';
import {
KeyValuePair,
KeyValuePairType,
} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service';
import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service';
import { User } from 'src/engine/core-modules/user/user.entity';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
describe('ConfigStorageService', () => {
let service: ConfigStorageService;
let keyValuePairRepository: Repository<KeyValuePair>;
let configValueConverter: ConfigValueConverterService;
const createMockKeyValuePair = (
key: string,
value: string,
): KeyValuePair => ({
id: '1',
key,
value: value as unknown as JSON,
type: KeyValuePairType.CONFIG_VARIABLE,
userId: null,
workspaceId: null,
user: null as unknown as User,
workspace: null as unknown as Workspace,
createdAt: new Date(),
updatedAt: new Date(),
textValueDeprecated: null,
deletedAt: null,
});
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
ConfigStorageService,
{
provide: ConfigValueConverterService,
useValue: {
convertDbValueToAppValue: jest.fn(),
convertAppValueToDbValue: jest.fn(),
},
},
ConfigVariables,
{
provide: getRepositoryToken(KeyValuePair, 'core'),
useValue: {
findOne: jest.fn(),
find: jest.fn(),
update: jest.fn(),
insert: jest.fn(),
delete: jest.fn(),
},
},
],
}).compile();
service = module.get<ConfigStorageService>(ConfigStorageService);
keyValuePairRepository = module.get<Repository<KeyValuePair>>(
getRepositoryToken(KeyValuePair, 'core'),
);
configValueConverter = module.get<ConfigValueConverterService>(
ConfigValueConverterService,
);
jest.clearAllMocks();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('get', () => {
it('should return undefined when key not found', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null);
const result = await service.get(key);
expect(result).toBeUndefined();
expect(keyValuePairRepository.findOne).toHaveBeenCalledWith({
where: {
type: KeyValuePairType.CONFIG_VARIABLE,
key: key as string,
userId: IsNull(),
workspaceId: IsNull(),
},
});
});
it('should return converted value when key found', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const storedValue = 'true';
const convertedValue = true;
const mockRecord = createMockKeyValuePair(key as string, storedValue);
jest
.spyOn(keyValuePairRepository, 'findOne')
.mockResolvedValue(mockRecord);
(
configValueConverter.convertDbValueToAppValue as jest.Mock
).mockReturnValue(convertedValue);
const result = await service.get(key);
expect(result).toBe(convertedValue);
expect(
configValueConverter.convertDbValueToAppValue,
).toHaveBeenCalledWith(storedValue, key);
});
it('should handle conversion errors', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const error = new Error('Conversion error');
const mockRecord = createMockKeyValuePair(key as string, 'invalid');
jest
.spyOn(keyValuePairRepository, 'findOne')
.mockResolvedValue(mockRecord);
(
configValueConverter.convertDbValueToAppValue as jest.Mock
).mockImplementation(() => {
throw error;
});
await expect(service.get(key)).rejects.toThrow('Conversion error');
});
});
describe('set', () => {
it('should update existing record', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const value = true;
const storedValue = 'true';
const mockRecord = createMockKeyValuePair(key as string, 'false');
jest
.spyOn(keyValuePairRepository, 'findOne')
.mockResolvedValue(mockRecord);
(
configValueConverter.convertAppValueToDbValue as jest.Mock
).mockReturnValue(storedValue);
await service.set(key, value);
expect(keyValuePairRepository.update).toHaveBeenCalledWith(
{ id: '1' },
{ value: storedValue },
);
});
it('should insert new record', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const value = true;
const storedValue = 'true';
jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null);
(
configValueConverter.convertAppValueToDbValue as jest.Mock
).mockReturnValue(storedValue);
await service.set(key, value);
expect(keyValuePairRepository.insert).toHaveBeenCalledWith({
key: key as string,
value: storedValue,
userId: null,
workspaceId: null,
type: KeyValuePairType.CONFIG_VARIABLE,
});
});
it('should handle conversion errors', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const value = true;
const error = new Error('Conversion error');
(
configValueConverter.convertAppValueToDbValue as jest.Mock
).mockImplementation(() => {
throw error;
});
await expect(service.set(key, value)).rejects.toThrow('Conversion error');
});
});
describe('delete', () => {
it('should delete record', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
await service.delete(key);
expect(keyValuePairRepository.delete).toHaveBeenCalledWith({
type: KeyValuePairType.CONFIG_VARIABLE,
key: key as string,
userId: IsNull(),
workspaceId: IsNull(),
});
});
it('should handle delete errors', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const error = new Error('Delete error');
jest.spyOn(keyValuePairRepository, 'delete').mockRejectedValue(error);
await expect(service.delete(key)).rejects.toThrow('Delete error');
});
});
describe('loadAll', () => {
it('should load and convert all config variables', async () => {
const configVars: KeyValuePair[] = [
createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'),
createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'),
];
jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars);
(
configValueConverter.convertDbValueToAppValue as jest.Mock
).mockImplementation((value, key) => {
if (key === 'AUTH_PASSWORD_ENABLED') return true;
if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com';
return value;
});
const result = await service.loadAll();
expect(result.size).toBe(2);
expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe(
true,
);
expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe(
'test@example.com',
);
});
it('should skip invalid values but continue processing', async () => {
const configVars: KeyValuePair[] = [
createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'invalid'),
createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'),
];
jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars);
(configValueConverter.convertDbValueToAppValue as jest.Mock)
.mockImplementationOnce(() => {
throw new Error('Invalid value');
})
.mockImplementationOnce((value) => value);
const result = await service.loadAll();
expect(result.size).toBe(1);
expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe(
'test@example.com',
);
});
it('should handle find errors', async () => {
const error = new Error('Find error');
jest.spyOn(keyValuePairRepository, 'find').mockRejectedValue(error);
await expect(service.loadAll()).rejects.toThrow('Find error');
});
describe('Null Value Handling', () => {
it('should handle null values in loadAll', async () => {
const configVars: KeyValuePair[] = [
{
...createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'),
value: null as unknown as JSON,
},
createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'),
];
jest
.spyOn(keyValuePairRepository, 'find')
.mockResolvedValue(configVars);
(
configValueConverter.convertDbValueToAppValue as jest.Mock
).mockImplementation((value) => {
if (value === null) throw new Error('Null value');
return value;
});
const result = await service.loadAll();
expect(result.size).toBe(1);
expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe(
'test@example.com',
);
expect(
configValueConverter.convertDbValueToAppValue,
).toHaveBeenCalledTimes(1); // Only called for non-null value
});
});
});
describe('Edge Cases and Additional Scenarios', () => {
describe('Type Safety', () => {
it('should enforce correct types for boolean config variables', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const invalidValue = 'not-a-boolean';
const mockRecord = createMockKeyValuePair(key as string, invalidValue);
jest
.spyOn(keyValuePairRepository, 'findOne')
.mockResolvedValue(mockRecord);
(
configValueConverter.convertDbValueToAppValue as jest.Mock
).mockImplementation(() => {
throw new Error('Invalid boolean value');
});
await expect(service.get(key)).rejects.toThrow('Invalid boolean value');
});
it('should enforce correct types for string config variables', async () => {
const key = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables;
const invalidValue = '123'; // Not a valid email
const mockRecord = createMockKeyValuePair(key as string, invalidValue);
jest
.spyOn(keyValuePairRepository, 'findOne')
.mockResolvedValue(mockRecord);
(
configValueConverter.convertDbValueToAppValue as jest.Mock
).mockImplementation(() => {
throw new Error('Invalid string value');
});
await expect(service.get(key)).rejects.toThrow('Invalid string value');
});
});
describe('Concurrent Operations', () => {
it('should handle concurrent get/set operations', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const initialValue = true;
const newValue = false;
const initialRecord = createMockKeyValuePair(key as string, 'true');
const updatedRecord = createMockKeyValuePair(key as string, 'false');
jest
.spyOn(keyValuePairRepository, 'findOne')
.mockResolvedValueOnce(initialRecord)
.mockResolvedValueOnce(initialRecord)
.mockResolvedValueOnce(updatedRecord);
(configValueConverter.convertDbValueToAppValue as jest.Mock)
.mockReturnValueOnce(initialValue)
.mockReturnValueOnce(newValue);
(
configValueConverter.convertAppValueToDbValue as jest.Mock
).mockReturnValueOnce('false');
const firstGet = service.get(key);
const setOperation = service.set(key, newValue);
const secondGet = service.get(key);
const [firstResult, , secondResult] = await Promise.all([
firstGet,
setOperation,
secondGet,
]);
expect(firstResult).toBe(initialValue);
expect(secondResult).toBe(newValue);
});
it('should handle concurrent delete operations', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
jest
.spyOn(keyValuePairRepository, 'delete')
.mockResolvedValueOnce({ affected: 1 } as DeleteResult)
.mockResolvedValueOnce({ affected: 0 } as DeleteResult);
const firstDelete = service.delete(key);
const secondDelete = service.delete(key);
await Promise.all([firstDelete, secondDelete]);
expect(keyValuePairRepository.delete).toHaveBeenCalledTimes(2);
});
});
describe('Database Connection Issues', () => {
it('should handle database connection failures in get', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const error = new Error('Database connection failed');
jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error);
await expect(service.get(key)).rejects.toThrow(
'Database connection failed',
);
});
it('should handle database connection failures in set', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const value = true;
const error = new Error('Database connection failed');
(
configValueConverter.convertAppValueToDbValue as jest.Mock
).mockReturnValue('true');
jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error);
await expect(service.set(key, value)).rejects.toThrow(
'Database connection failed',
);
});
it('should handle database connection failures in delete', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const error = new Error('Database connection failed');
jest.spyOn(keyValuePairRepository, 'delete').mockRejectedValue(error);
await expect(service.delete(key)).rejects.toThrow(
'Database connection failed',
);
});
it('should handle database connection failures in loadAll', async () => {
const error = new Error('Database connection failed');
jest.spyOn(keyValuePairRepository, 'find').mockRejectedValue(error);
await expect(service.loadAll()).rejects.toThrow(
'Database connection failed',
);
});
it('should handle database operation timeouts', async () => {
const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables;
const error = new Error('Database operation timed out');
let rejectPromise: ((error: Error) => void) | undefined;
const timeoutPromise = new Promise<KeyValuePair | null>((_, reject) => {
rejectPromise = reject;
});
jest
.spyOn(keyValuePairRepository, 'findOne')
.mockReturnValue(timeoutPromise);
const promise = service.get(key);
// Simulate timeout by rejecting the promise
if (!rejectPromise) {
throw new Error('Reject function not assigned');
}
rejectPromise(error);
await expect(promise).rejects.toThrow('Database operation timed out');
});
});
});
});

View File

@ -0,0 +1,165 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { FindOptionsWhere, IsNull, Repository } from 'typeorm';
import {
KeyValuePair,
KeyValuePairType,
} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service';
import { ConfigStorageInterface } from './interfaces/config-storage.interface';
@Injectable()
export class ConfigStorageService implements ConfigStorageInterface {
private readonly logger = new Logger(ConfigStorageService.name);
constructor(
@InjectRepository(KeyValuePair, 'core')
private readonly keyValuePairRepository: Repository<KeyValuePair>,
private readonly configValueConverter: ConfigValueConverterService,
) {}
private getConfigVariableWhereClause(
key?: string,
): FindOptionsWhere<KeyValuePair> {
return {
type: KeyValuePairType.CONFIG_VARIABLE,
...(key ? { key } : {}),
userId: IsNull(),
workspaceId: IsNull(),
};
}
async get<T extends keyof ConfigVariables>(
key: T,
): Promise<ConfigVariables[T] | undefined> {
try {
const result = await this.keyValuePairRepository.findOne({
where: this.getConfigVariableWhereClause(key as string),
});
if (result === null) {
return undefined;
}
try {
this.logger.debug(
`Fetching config for ${key as string} in database: ${result?.value}`,
);
return this.configValueConverter.convertDbValueToAppValue(
result.value,
key,
);
} catch (error) {
this.logger.error(
`Failed to convert value to app type for key ${key as string}`,
error,
);
throw error;
}
} catch (error) {
this.logger.error(`Failed to get config for ${key as string}`, error);
throw error;
}
}
async set<T extends keyof ConfigVariables>(
key: T,
value: ConfigVariables[T],
): Promise<void> {
try {
let processedValue;
try {
processedValue =
this.configValueConverter.convertAppValueToDbValue(value);
} catch (error) {
this.logger.error(
`Failed to convert value to storage type for key ${key as string}`,
error,
);
throw error;
}
const existingRecord = await this.keyValuePairRepository.findOne({
where: this.getConfigVariableWhereClause(key as string),
});
if (existingRecord) {
await this.keyValuePairRepository.update(
{ id: existingRecord.id },
{ value: processedValue },
);
} else {
await this.keyValuePairRepository.insert({
key: key as string,
value: processedValue,
userId: null,
workspaceId: null,
type: KeyValuePairType.CONFIG_VARIABLE,
});
}
} catch (error) {
this.logger.error(`Failed to set config for ${key as string}`, error);
throw error;
}
}
async delete<T extends keyof ConfigVariables>(key: T): Promise<void> {
try {
await this.keyValuePairRepository.delete(
this.getConfigVariableWhereClause(key as string),
);
} catch (error) {
this.logger.error(`Failed to delete config for ${key as string}`, error);
throw error;
}
}
async loadAll(): Promise<
Map<keyof ConfigVariables, ConfigVariables[keyof ConfigVariables]>
> {
try {
const configVars = await this.keyValuePairRepository.find({
where: this.getConfigVariableWhereClause(),
});
const result = new Map<
keyof ConfigVariables,
ConfigVariables[keyof ConfigVariables]
>();
for (const configVar of configVars) {
if (configVar.value !== null) {
const key = configVar.key as keyof ConfigVariables;
try {
const value = this.configValueConverter.convertDbValueToAppValue(
configVar.value,
key,
);
if (value !== undefined) {
result.set(key, value);
}
} catch (error) {
this.logger.error(
`Failed to convert value to app type for key ${key as string}`,
error,
);
continue;
}
}
}
return result;
} catch (error) {
this.logger.error('Failed to load all config variables', error);
throw error;
}
}
}

View File

@ -0,0 +1,18 @@
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
export interface ConfigStorageInterface {
get<T extends keyof ConfigVariables>(
key: T,
): Promise<ConfigVariables[T] | undefined>;
set<T extends keyof ConfigVariables>(
key: T,
value: ConfigVariables[T],
): Promise<void>;
delete<T extends keyof ConfigVariables>(key: T): Promise<void>;
loadAll(): Promise<
Map<keyof ConfigVariables, ConfigVariables[keyof ConfigVariables]>
>;
}

View File

@ -1,21 +1,48 @@
import { Global, Module } from '@nestjs/common';
import { DynamicModule, Global, Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { validate } from 'src/engine/core-modules/twenty-config/config-variables';
import {
ConfigVariables,
validate,
} from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants';
import { DatabaseConfigModule } from 'src/engine/core-modules/twenty-config/drivers/database-config.module';
import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver';
import { ConfigurableModuleClass } from 'src/engine/core-modules/twenty-config/twenty-config.module-definition';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
@Global()
@Module({
imports: [
ConfigModule.forRoot({
isGlobal: true,
expandVariables: true,
validate,
envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env',
}),
],
providers: [TwentyConfigService],
exports: [TwentyConfigService],
})
export class TwentyConfigModule extends ConfigurableModuleClass {}
@Module({})
export class TwentyConfigModule extends ConfigurableModuleClass {
static forRoot(): DynamicModule {
const isConfigVariablesInDbEnabled =
process.env.IS_CONFIG_VARIABLES_IN_DB_ENABLED === 'true';
const imports = [
ConfigModule.forRoot({
isGlobal: true,
expandVariables: true,
validate,
envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env',
}),
];
if (isConfigVariablesInDbEnabled) {
imports.push(DatabaseConfigModule.forRoot());
}
return {
module: TwentyConfigModule,
imports,
providers: [
TwentyConfigService,
EnvironmentConfigDriver,
{
provide: CONFIG_VARIABLES_INSTANCE_TOKEN,
useValue: new ConfigVariables(),
},
],
exports: [TwentyConfigService],
};
}
}

View File

@ -2,81 +2,470 @@ import { ConfigService } from '@nestjs/config';
import { Test, TestingModule } from '@nestjs/testing';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver';
import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver';
import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum';
import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util';
import { TypedReflect } from 'src/utils/typed-reflect';
jest.mock('src/utils/typed-reflect', () => ({
TypedReflect: {
getMetadata: jest.fn(),
defineMetadata: jest.fn(),
},
}));
jest.mock(
'src/engine/core-modules/twenty-config/constants/config-variables-masking-config',
() => ({
CONFIG_VARIABLES_MASKING_CONFIG: {
SENSITIVE_VAR: {
strategy: 'LAST_N_CHARS',
chars: 5,
},
},
}),
);
jest.mock(
'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util',
() => ({
isEnvOnlyConfigVar: jest.fn(),
}),
);
type TwentyConfigServicePrivateProps = {
isDatabaseDriverActive: boolean;
};
const mockConfigVarMetadata = {
TEST_VAR: {
group: ConfigVariablesGroup.GoogleAuth,
description: 'Test variable',
isEnvOnly: false,
},
ENV_ONLY_VAR: {
group: ConfigVariablesGroup.StorageConfig,
description: 'Environment only variable',
isEnvOnly: true,
},
SENSITIVE_VAR: {
group: ConfigVariablesGroup.Logging,
description: 'Sensitive variable',
isSensitive: true,
},
};
// Setup with database driver
const setupTestModule = async (isDatabaseConfigEnabled = true) => {
const configServiceMock = {
get: jest.fn().mockImplementation((key) => {
if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') {
return isDatabaseConfigEnabled ? 'true' : 'false';
}
return undefined;
}),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
TwentyConfigService,
{
provide: DatabaseConfigDriver,
useValue: {
get: jest.fn(),
update: jest.fn(),
getCacheInfo: jest.fn(),
},
},
{
provide: EnvironmentConfigDriver,
useValue: {
get: jest.fn().mockImplementation((key) => {
return configServiceMock.get(key);
}),
},
},
{
provide: ConfigService,
useValue: configServiceMock,
},
],
}).compile();
return {
service: module.get<TwentyConfigService>(TwentyConfigService),
databaseConfigDriver:
module.get<DatabaseConfigDriver>(DatabaseConfigDriver),
environmentConfigDriver: module.get<EnvironmentConfigDriver>(
EnvironmentConfigDriver,
),
configService: module.get<ConfigService>(ConfigService),
};
};
// Setup without database driver
const setupTestModuleWithoutDb = async () => {
const configServiceMock = {
get: jest.fn().mockImplementation((key) => {
if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') {
return 'false';
}
return undefined;
}),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
TwentyConfigService,
{
provide: EnvironmentConfigDriver,
useValue: {
get: jest.fn().mockImplementation((key) => {
return configServiceMock.get(key);
}),
},
},
{
provide: ConfigService,
useValue: configServiceMock,
},
],
}).compile();
return {
service: module.get<TwentyConfigService>(TwentyConfigService),
environmentConfigDriver: module.get<EnvironmentConfigDriver>(
EnvironmentConfigDriver,
),
configService: module.get<ConfigService>(ConfigService),
};
};
const setPrivateProps = (
service: TwentyConfigService,
props: Partial<TwentyConfigServicePrivateProps>,
) => {
Object.entries(props).forEach(([key, value]) => {
Object.defineProperty(service, key, {
value,
writable: true,
});
});
};
describe('TwentyConfigService', () => {
let service: TwentyConfigService;
let configService: ConfigService;
let databaseConfigDriver: DatabaseConfigDriver;
let environmentConfigDriver: EnvironmentConfigDriver;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
TwentyConfigService,
{
provide: ConfigService,
useValue: {
get: jest.fn(),
},
},
],
}).compile();
const testModule = await setupTestModule(true);
service = module.get<TwentyConfigService>(TwentyConfigService);
configService = module.get<ConfigService>(ConfigService);
service = testModule.service;
databaseConfigDriver = testModule.databaseConfigDriver;
environmentConfigDriver = testModule.environmentConfigDriver;
Reflect.defineMetadata('config-variables', {}, ConfigVariables);
(TypedReflect.getMetadata as jest.Mock).mockReturnValue(
mockConfigVarMetadata,
);
});
afterEach(() => {
jest.clearAllMocks();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('getAll()', () => {
it('should return empty object when no config variables are defined', () => {
const result = service.getAll();
describe('constructor', () => {
it('should set isDatabaseDriverActive to false when database config is disabled', async () => {
const { service, environmentConfigDriver } =
await setupTestModuleWithoutDb();
expect(result).toEqual({});
expect(environmentConfigDriver.get).toHaveBeenCalledWith(
'IS_CONFIG_VARIABLES_IN_DB_ENABLED',
);
expect(service.getCacheInfo().usingDatabaseDriver).toBe(false);
});
it('should return config variables with their metadata', () => {
const mockMetadata = {
TEST_VAR: {
title: 'Test Var',
description: 'Test Description',
},
};
it('should set isDatabaseDriverActive to true when database config is enabled and driver is available', async () => {
const { service, environmentConfigDriver } = await setupTestModule(true);
Reflect.defineMetadata('config-variables', mockMetadata, ConfigVariables);
expect(environmentConfigDriver.get).toHaveBeenCalledWith(
'IS_CONFIG_VARIABLES_IN_DB_ENABLED',
);
jest.spyOn(configService, 'get').mockReturnValue('test-value');
expect(service.getCacheInfo().usingDatabaseDriver).toBe(true);
});
});
describe('get', () => {
const key = 'TEST_VAR' as keyof ConfigVariables;
const expectedValue = 'test value';
beforeEach(() => {
(isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false);
});
it('should use environment driver for environment-only variables', () => {
(isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true);
jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(expectedValue);
const result = service.get(key);
expect(result).toBe(expectedValue);
expect(environmentConfigDriver.get).toHaveBeenCalledWith(key);
});
it('should return undefined when key does not exist in any driver', () => {
const nonExistentKey = 'NON_EXISTENT_KEY' as keyof ConfigVariables;
jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(undefined);
jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(undefined);
setPrivateProps(service, { isDatabaseDriverActive: true });
const result = service.get(nonExistentKey);
expect(result).toBeUndefined();
expect(databaseConfigDriver.get).toHaveBeenCalledWith(nonExistentKey);
expect(environmentConfigDriver.get).toHaveBeenCalledWith(nonExistentKey);
});
it('should use database driver when isDatabaseDriverActive is true and value is found', () => {
jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue);
setPrivateProps(service, { isDatabaseDriverActive: true });
jest.clearAllMocks();
const result = service.get(key);
expect(result).toBe(expectedValue);
expect(databaseConfigDriver.get).toHaveBeenCalledWith(key);
expect(environmentConfigDriver.get).not.toHaveBeenCalled();
});
it('should fall back to environment driver when database driver is active but value is not found', () => {
const envValue = 'env value';
jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(undefined);
jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(envValue);
setPrivateProps(service, { isDatabaseDriverActive: true });
const result = service.get(key);
expect(result).toBe(envValue);
expect(databaseConfigDriver.get).toHaveBeenCalledWith(key);
expect(environmentConfigDriver.get).toHaveBeenCalledWith(key);
});
it('should use environment driver when isDatabaseDriverActive is false', () => {
jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(expectedValue);
setPrivateProps(service, { isDatabaseDriverActive: false });
const result = service.get(key);
expect(result).toBe(expectedValue);
expect(environmentConfigDriver.get).toHaveBeenCalledWith(key);
expect(databaseConfigDriver.get).not.toHaveBeenCalled();
});
});
describe('update', () => {
it('should throw error when database driver is not active', async () => {
setPrivateProps(service, { isDatabaseDriverActive: false });
await expect(
service.update('TEST_VAR' as keyof ConfigVariables, 'new value'),
).rejects.toThrow(
'Database configuration is disabled or unavailable, cannot update configuration',
);
});
it('should throw error when updating environment-only variable', async () => {
setPrivateProps(service, { isDatabaseDriverActive: true });
(TypedReflect.getMetadata as jest.Mock).mockReturnValue({
ENV_ONLY_VAR: { isEnvOnly: true },
});
await expect(
service.update('ENV_ONLY_VAR' as keyof ConfigVariables, 'new value'),
).rejects.toThrow(
'Cannot update environment-only variable: ENV_ONLY_VAR',
);
});
it('should update config when database driver is active', async () => {
const key = 'TEST_VAR' as keyof ConfigVariables;
const newValue = 'new value';
setPrivateProps(service, { isDatabaseDriverActive: true });
jest.spyOn(databaseConfigDriver, 'update').mockResolvedValue(undefined);
await service.update(key, newValue);
expect(databaseConfigDriver.update).toHaveBeenCalledWith(key, newValue);
});
it('should propagate errors from database driver', async () => {
const error = new Error('Database error');
setPrivateProps(service, { isDatabaseDriverActive: true });
jest.spyOn(databaseConfigDriver, 'update').mockRejectedValue(error);
await expect(
service.update('TEST_VAR' as keyof ConfigVariables, 'new value'),
).rejects.toThrow(error);
});
});
describe('getMetadata', () => {
it('should return metadata for a config variable', () => {
const result = service.getMetadata('TEST_VAR' as keyof ConfigVariables);
expect(result).toEqual(mockConfigVarMetadata.TEST_VAR);
});
it('should return undefined when metadata does not exist', () => {
const result = service.getMetadata(
'UNKNOWN_VAR' as keyof ConfigVariables,
);
expect(result).toBeUndefined();
});
});
describe('getAll', () => {
const setupDriverMocks = () => {
jest
.spyOn(environmentConfigDriver, 'get')
.mockImplementation((key: keyof ConfigVariables) => {
const keyStr = String(key);
const values = {
TEST_VAR: 'env test value',
ENV_ONLY_VAR: 'env only value',
SENSITIVE_VAR: 'sensitive_data_123',
};
return values[keyStr] || undefined;
});
jest
.spyOn(databaseConfigDriver, 'get')
.mockImplementation((key: keyof ConfigVariables) => {
const keyStr = String(key);
if (mockConfigVarMetadata[keyStr]?.isEnvOnly) {
return environmentConfigDriver.get(key);
}
const values = {
TEST_VAR: 'db test value',
SENSITIVE_VAR: 'sensitive_data_123',
};
return values[keyStr] || undefined;
});
};
beforeEach(() => {
setupDriverMocks();
});
it('should return all config variables with environment source when database driver is not active', () => {
setPrivateProps(service, {
isDatabaseDriverActive: false,
});
const result = service.getAll();
expect(result).toEqual({
TEST_VAR: {
value: 'test-value',
metadata: mockMetadata.TEST_VAR,
value: 'env test value',
metadata: mockConfigVarMetadata.TEST_VAR,
source: ConfigSource.ENVIRONMENT,
},
ENV_ONLY_VAR: {
value: 'env only value',
metadata: mockConfigVarMetadata.ENV_ONLY_VAR,
source: ConfigSource.ENVIRONMENT,
},
SENSITIVE_VAR: {
value: expect.any(String),
metadata: mockConfigVarMetadata.SENSITIVE_VAR,
source: ConfigSource.ENVIRONMENT,
},
});
expect(result.SENSITIVE_VAR.value).toBe('********a_123');
});
it('should mask sensitive data according to masking config', () => {
const mockMetadata = {
APP_SECRET: {
title: 'App Secret',
description: 'Application secret key',
sensitive: true,
},
};
Reflect.defineMetadata('config-variables', mockMetadata, ConfigVariables);
jest.spyOn(configService, 'get').mockReturnValue('super-secret-value');
it('should return config variables with database source when database driver is active', () => {
setPrivateProps(service, {
isDatabaseDriverActive: true,
});
const result = service.getAll();
expect(result.APP_SECRET.value).not.toBe('super-secret-value');
expect(result.APP_SECRET.value).toMatch(/^\*+[a-zA-Z0-9]{5}$/);
expect(result.TEST_VAR).toEqual({
value: 'db test value',
metadata: mockConfigVarMetadata.TEST_VAR,
source: ConfigSource.DATABASE,
});
expect(result.ENV_ONLY_VAR).toEqual({
value: 'env only value',
metadata: mockConfigVarMetadata.ENV_ONLY_VAR,
source: ConfigSource.ENVIRONMENT,
});
expect(result.SENSITIVE_VAR).toEqual({
value: '********a_123',
metadata: mockConfigVarMetadata.SENSITIVE_VAR,
source: ConfigSource.DATABASE,
});
});
});
describe('getCacheInfo', () => {
it('should return basic info when database driver is not active', () => {
setPrivateProps(service, {
isDatabaseDriverActive: false,
});
const result = service.getCacheInfo();
expect(result).toEqual({
usingDatabaseDriver: false,
});
});
it('should return cache stats when database driver is active', () => {
const cacheStats = {
foundConfigValues: 2,
knownMissingKeys: 1,
cacheKeys: ['TEST_VAR', 'SENSITIVE_VAR'],
};
setPrivateProps(service, {
isDatabaseDriverActive: true,
});
jest
.spyOn(databaseConfigDriver, 'getCacheInfo')
.mockReturnValue(cacheStats);
const result = service.getCacheInfo();
expect(result).toEqual({
usingDatabaseDriver: true,
cacheStats,
});
});
});
});

View File

@ -1,22 +1,107 @@
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Injectable, Logger, Optional } from '@nestjs/common';
import { isString } from 'class-validator';
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config';
import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator';
import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver';
import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver';
import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum';
import { ConfigVariablesMaskingStrategies } from 'src/engine/core-modules/twenty-config/enums/config-variables-masking-strategies.enum';
import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty-config/utils/config-variable-mask-sensitive-data.util';
import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util';
import { TypedReflect } from 'src/utils/typed-reflect';
@Injectable()
export class TwentyConfigService {
constructor(private readonly configService: ConfigService) {}
private readonly logger = new Logger(TwentyConfigService.name);
private readonly isDatabaseDriverActive: boolean;
constructor(
private readonly environmentConfigDriver: EnvironmentConfigDriver,
@Optional() private readonly databaseConfigDriver: DatabaseConfigDriver,
) {
const isConfigVariablesInDbEnabled = this.environmentConfigDriver.get(
'IS_CONFIG_VARIABLES_IN_DB_ENABLED',
);
this.isDatabaseDriverActive =
isConfigVariablesInDbEnabled && !!this.databaseConfigDriver;
this.logger.log(
`Database configuration is ${isConfigVariablesInDbEnabled ? 'enabled' : 'disabled'}`,
);
if (isConfigVariablesInDbEnabled && !this.databaseConfigDriver) {
this.logger.warn(
'Database config is enabled but driver is not available. Using environment variables only.',
);
}
if (this.isDatabaseDriverActive) {
this.logger.log('Using database configuration driver');
// The database driver will load config variables asynchronously via its onModuleInit lifecycle hook
// In the meantime, we'll use the environment driver -- fallback
} else {
this.logger.log('Using environment variables only for configuration');
}
}
get<T extends keyof ConfigVariables>(key: T): ConfigVariables[T] {
return this.configService.get<ConfigVariables[T]>(
key,
new ConfigVariables()[key],
);
if (isEnvOnlyConfigVar(key)) {
return this.environmentConfigDriver.get(key);
}
if (this.isDatabaseDriverActive) {
const cachedValueFromDb = this.databaseConfigDriver.get(key);
if (cachedValueFromDb !== undefined) {
return cachedValueFromDb;
}
return this.environmentConfigDriver.get(key);
}
return this.environmentConfigDriver.get(key);
}
async update<T extends keyof ConfigVariables>(
key: T,
value: ConfigVariables[T],
): Promise<void> {
if (!this.isDatabaseDriverActive) {
throw new Error(
'Database configuration is disabled or unavailable, cannot update configuration',
);
}
const metadata =
TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {};
const envMetadata = metadata[key];
if (envMetadata?.isEnvOnly) {
throw new Error(
`Cannot update environment-only variable: ${key as string}`,
);
}
try {
await this.databaseConfigDriver.update(key, value);
this.logger.debug(`Updated config variable: ${key as string}`);
} catch (error) {
this.logger.error(`Failed to update config for ${key as string}`, error);
throw error;
}
}
getMetadata(
key: keyof ConfigVariables,
): ConfigVariablesMetadataOptions | undefined {
const metadata =
TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {};
return metadata[key];
}
getAll(): Record<
@ -24,6 +109,7 @@ export class TwentyConfigService {
{
value: ConfigVariables[keyof ConfigVariables];
metadata: ConfigVariablesMetadataOptions;
source: ConfigSource;
}
> {
const result: Record<
@ -31,6 +117,7 @@ export class TwentyConfigService {
{
value: ConfigVariables[keyof ConfigVariables];
metadata: ConfigVariablesMetadataOptions;
source: ConfigSource;
}
> = {};
@ -39,12 +126,23 @@ export class TwentyConfigService {
TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {};
Object.entries(metadata).forEach(([key, envMetadata]) => {
let value =
this.configService.get(key) ??
configVars[key as keyof ConfigVariables] ??
'';
let value = this.get(key as keyof ConfigVariables) ?? '';
let source = ConfigSource.ENVIRONMENT;
if (typeof value === 'string' && key in CONFIG_VARIABLES_MASKING_CONFIG) {
if (!this.isDatabaseDriverActive || envMetadata.isEnvOnly) {
if (value === configVars[key as keyof ConfigVariables]) {
source = ConfigSource.DEFAULT;
}
} else {
const dbValue = value;
source =
dbValue !== configVars[key as keyof ConfigVariables]
? ConfigSource.DATABASE
: ConfigSource.DEFAULT;
}
if (isString(value) && key in CONFIG_VARIABLES_MASKING_CONFIG) {
const varMaskingConfig =
CONFIG_VARIABLES_MASKING_CONFIG[
key as keyof typeof CONFIG_VARIABLES_MASKING_CONFIG
@ -65,9 +163,32 @@ export class TwentyConfigService {
result[key] = {
value,
metadata: envMetadata,
source,
};
});
return result;
}
getCacheInfo(): {
usingDatabaseDriver: boolean;
cacheStats?: {
foundConfigValues: number;
knownMissingKeys: number;
cacheKeys: string[];
};
} {
const result = {
usingDatabaseDriver: this.isDatabaseDriverActive,
};
if (this.isDatabaseDriverActive) {
return {
...result,
cacheStats: this.databaseConfigDriver.getCacheInfo(),
};
}
return result;
}
}

View File

@ -0,0 +1,3 @@
export type ConfigVariableOptions =
| readonly (string | number | boolean)[]
| Record<string, string>;

View File

@ -0,0 +1,6 @@
export type ConfigVariableType =
| 'boolean'
| 'number'
| 'array'
| 'string'
| 'enum';

View File

@ -0,0 +1,161 @@
import { Transform } from 'class-transformer';
import {
IsArray,
IsBoolean,
IsEnum,
IsNumber,
IsString,
} from 'class-validator';
import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util';
import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util';
jest.mock('class-transformer', () => ({
Transform: jest.fn(),
}));
jest.mock('class-validator', () => ({
IsBoolean: jest.fn().mockReturnValue(jest.fn()),
IsNumber: jest.fn().mockReturnValue(jest.fn()),
IsString: jest.fn().mockReturnValue(jest.fn()),
IsEnum: jest.fn().mockReturnValue(jest.fn()),
IsArray: jest.fn().mockReturnValue(jest.fn()),
}));
jest.mock(
'src/engine/core-modules/twenty-config/utils/config-transformers.util',
() => ({
configTransformers: {
boolean: jest.fn(),
number: jest.fn(),
},
}),
);
describe('applyBasicValidators', () => {
const mockTarget = {};
const mockPropertyKey = 'testProperty';
beforeEach(() => {
jest.clearAllMocks();
});
describe('boolean type', () => {
it('should apply boolean transformers and validators', () => {
let capturedTransformFn;
(Transform as jest.Mock).mockImplementation((transformFn) => {
capturedTransformFn = transformFn;
return jest.fn();
});
applyBasicValidators('boolean', mockTarget, mockPropertyKey);
expect(Transform).toHaveBeenCalled();
expect(IsBoolean).toHaveBeenCalled();
const transformFn = capturedTransformFn;
const mockTransformParams = { value: 'true' };
(configTransformers.boolean as jest.Mock).mockReturnValueOnce(true);
const result1 = transformFn(mockTransformParams);
expect(configTransformers.boolean).toHaveBeenCalledWith('true');
expect(result1).toBe(true);
(configTransformers.boolean as jest.Mock).mockReturnValueOnce(undefined);
const result2 = transformFn(mockTransformParams);
expect(result2).toBe('true');
});
});
describe('number type', () => {
it('should apply number transformers and validators', () => {
let capturedTransformFn;
(Transform as jest.Mock).mockImplementation((transformFn) => {
capturedTransformFn = transformFn;
return jest.fn();
});
applyBasicValidators('number', mockTarget, mockPropertyKey);
expect(Transform).toHaveBeenCalled();
expect(IsNumber).toHaveBeenCalled();
const transformFn = capturedTransformFn;
const mockTransformParams = { value: '42' };
(configTransformers.number as jest.Mock).mockReturnValueOnce(42);
const result1 = transformFn(mockTransformParams);
expect(configTransformers.number).toHaveBeenCalledWith('42');
expect(result1).toBe(42);
(configTransformers.number as jest.Mock).mockReturnValueOnce(undefined);
const result2 = transformFn(mockTransformParams);
expect(result2).toBe('42');
});
});
describe('string type', () => {
it('should apply string validator', () => {
applyBasicValidators('string', mockTarget, mockPropertyKey);
expect(IsString).toHaveBeenCalled();
expect(Transform).not.toHaveBeenCalled(); // String doesn't need a transform
});
});
describe('enum type', () => {
it('should apply enum validator with string array options', () => {
const enumOptions = ['option1', 'option2', 'option3'];
applyBasicValidators('enum', mockTarget, mockPropertyKey, enumOptions);
expect(IsEnum).toHaveBeenCalledWith(enumOptions);
expect(Transform).not.toHaveBeenCalled(); // Enum doesn't need a transform
});
it('should apply enum validator with enum object options', () => {
enum TestEnum {
Option1 = 'value1',
Option2 = 'value2',
Option3 = 'value3',
}
applyBasicValidators('enum', mockTarget, mockPropertyKey, TestEnum);
expect(IsEnum).toHaveBeenCalledWith(TestEnum);
expect(Transform).not.toHaveBeenCalled(); // Enum doesn't need a transform
});
it('should not apply enum validator without options', () => {
applyBasicValidators('enum', mockTarget, mockPropertyKey);
expect(IsEnum).not.toHaveBeenCalled();
expect(Transform).not.toHaveBeenCalled();
});
});
describe('array type', () => {
it('should apply array validator', () => {
applyBasicValidators('array', mockTarget, mockPropertyKey);
expect(IsArray).toHaveBeenCalled();
expect(Transform).not.toHaveBeenCalled(); // Array doesn't need a transform
});
});
describe('unsupported type', () => {
it('should throw error for unsupported types', () => {
expect(() => {
applyBasicValidators('unsupported' as any, mockTarget, mockPropertyKey);
}).toThrow('Unsupported config variable type: unsupported');
});
});
});

View File

@ -0,0 +1,100 @@
import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util';
describe('configTransformers', () => {
describe('boolean', () => {
it('should handle true values correctly', () => {
expect(configTransformers.boolean(true)).toBe(true);
expect(configTransformers.boolean('true')).toBe(true);
expect(configTransformers.boolean('True')).toBe(true);
expect(configTransformers.boolean('yes')).toBe(true);
expect(configTransformers.boolean('on')).toBe(true);
expect(configTransformers.boolean('1')).toBe(true);
expect(configTransformers.boolean(1)).toBe(true);
});
it('should handle false values correctly', () => {
expect(configTransformers.boolean(false)).toBe(false);
expect(configTransformers.boolean('false')).toBe(false);
expect(configTransformers.boolean('False')).toBe(false);
expect(configTransformers.boolean('no')).toBe(false);
expect(configTransformers.boolean('off')).toBe(false);
expect(configTransformers.boolean('0')).toBe(false);
expect(configTransformers.boolean(0)).toBe(false);
});
it('should return undefined for invalid values', () => {
expect(configTransformers.boolean('invalid')).toBeUndefined();
expect(configTransformers.boolean('random_string')).toBeUndefined();
expect(configTransformers.boolean({})).toBeUndefined();
expect(configTransformers.boolean([])).toBeUndefined();
});
it('should handle null and undefined', () => {
expect(configTransformers.boolean(null)).toBeUndefined();
expect(configTransformers.boolean(undefined)).toBeUndefined();
});
});
describe('number', () => {
it('should handle valid number values', () => {
expect(configTransformers.number(42)).toBe(42);
expect(configTransformers.number('42')).toBe(42);
expect(configTransformers.number('-42')).toBe(-42);
expect(configTransformers.number('3.14')).toBe(3.14);
expect(configTransformers.number('0')).toBe(0);
});
it('should handle boolean values', () => {
expect(configTransformers.number(true)).toBe(1);
expect(configTransformers.number(false)).toBe(0);
});
it('should return undefined for invalid values', () => {
expect(configTransformers.number('invalid')).toBeUndefined();
expect(configTransformers.number('forty-two')).toBeUndefined();
expect(configTransformers.number({})).toBeUndefined();
expect(configTransformers.number([])).toBeUndefined();
});
it('should handle null and undefined', () => {
expect(configTransformers.number(null)).toBeUndefined();
expect(configTransformers.number(undefined)).toBeUndefined();
});
});
describe('string', () => {
it('should handle string values', () => {
expect(configTransformers.string('test')).toBe('test');
expect(configTransformers.string('')).toBe('');
});
it('should convert numbers to strings', () => {
expect(configTransformers.string(42)).toBe('42');
expect(configTransformers.string(0)).toBe('0');
expect(configTransformers.string(3.14)).toBe('3.14');
});
it('should convert booleans to strings', () => {
expect(configTransformers.string(true)).toBe('true');
expect(configTransformers.string(false)).toBe('false');
});
it('should convert arrays and objects to JSON strings', () => {
expect(configTransformers.string(['a', 'b', 'c'])).toBe('["a","b","c"]');
expect(configTransformers.string({ a: 1, b: 2 })).toBe('{"a":1,"b":2}');
});
it('should handle null and undefined', () => {
expect(configTransformers.string(null)).toBeUndefined();
expect(configTransformers.string(undefined)).toBeUndefined();
});
it('should handle failed JSON stringification', () => {
const circular: any = {};
circular.self = circular;
expect(configTransformers.string(circular)).toBeUndefined();
});
});
});

View File

@ -0,0 +1,56 @@
import { Transform } from 'class-transformer';
import {
IsArray,
IsBoolean,
IsEnum,
IsNumber,
IsString,
} from 'class-validator';
import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type';
import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type';
import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util';
export function applyBasicValidators(
type: ConfigVariableType,
target: object,
propertyKey: string,
options?: ConfigVariableOptions,
): void {
switch (type) {
case 'boolean':
Transform(({ value }) => {
const result = configTransformers.boolean(value);
return result !== undefined ? result : value;
})(target, propertyKey);
IsBoolean()(target, propertyKey);
break;
case 'number':
Transform(({ value }) => {
const result = configTransformers.number(value);
return result !== undefined ? result : value;
})(target, propertyKey);
IsNumber()(target, propertyKey);
break;
case 'string':
IsString()(target, propertyKey);
break;
case 'enum':
if (options) {
IsEnum(options)(target, propertyKey);
}
break;
case 'array':
IsArray()(target, propertyKey);
break;
default:
throw new Error(`Unsupported config variable type: ${type}`);
}
}

View File

@ -0,0 +1,79 @@
export const configTransformers = {
boolean: (value: unknown): boolean | undefined => {
if (value === null || value === undefined) {
return undefined;
}
if (typeof value === 'boolean') {
return value;
}
if (typeof value === 'number') {
return value !== 0;
}
if (typeof value === 'string') {
const lowerValue = value.toLowerCase();
if (['true', 'on', 'yes', '1'].includes(lowerValue)) {
return true;
}
if (['false', 'off', 'no', '0'].includes(lowerValue)) {
return false;
}
}
return undefined;
},
number: (value: unknown): number | undefined => {
if (value === null || value === undefined) {
return undefined;
}
if (typeof value === 'number') {
return value;
}
if (typeof value === 'string') {
const parsedNumber = parseFloat(value);
if (isNaN(parsedNumber)) {
return undefined;
}
return parsedNumber;
}
if (typeof value === 'boolean') {
return value ? 1 : 0;
}
return undefined;
},
string: (value: unknown): string | undefined => {
if (value === null || value === undefined) {
return undefined;
}
if (typeof value === 'string') {
return value;
}
if (typeof value === 'number' || typeof value === 'boolean') {
return String(value);
}
if (Array.isArray(value) || typeof value === 'object') {
try {
return JSON.stringify(value);
} catch {
return undefined;
}
}
return undefined;
},
};

View File

@ -0,0 +1,10 @@
import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables';
import { TypedReflect } from 'src/utils/typed-reflect';
export const isEnvOnlyConfigVar = (key: keyof ConfigVariables): boolean => {
const metadata =
TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {};
const envMetadata = metadata[key];
return !!envMetadata?.isEnvOnly;
};

View File

@ -28,7 +28,7 @@ xdescribe('Microsoft dev tests : get message list service', () => {
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
imports: [TwentyConfigModule.forRoot({})],
imports: [TwentyConfigModule.forRoot()],
providers: [
MicrosoftGetMessageListService,
MicrosoftClientProvider,
@ -118,7 +118,7 @@ xdescribe('Microsoft dev tests : get full message list service for folders', ()
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
imports: [TwentyConfigModule.forRoot({})],
imports: [TwentyConfigModule.forRoot()],
providers: [
MicrosoftGetMessageListService,
MicrosoftClientProvider,
@ -207,7 +207,7 @@ xdescribe('Microsoft dev tests : get partial message list service for folders',
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
imports: [TwentyConfigModule.forRoot({})],
imports: [TwentyConfigModule.forRoot()],
providers: [
MicrosoftGetMessageListService,
MicrosoftClientProvider,

View File

@ -23,7 +23,7 @@ xdescribe('Microsoft dev tests : get messages service', () => {
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
imports: [TwentyConfigModule.forRoot({})],
imports: [TwentyConfigModule.forRoot()],
providers: [
MicrosoftGetMessagesService,
MicrosoftHandleErrorService,

View File

@ -3,7 +3,7 @@ import { Test, TestingModule } from '@nestjs/testing';
import { ConnectedAccountProvider } from 'twenty-shared/types';
import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service';
import {
microsoftGraphBatchWithHtmlMessagesResponse,
@ -21,7 +21,6 @@ describe('Microsoft get messages service', () => {
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
imports: [TwentyConfigModule.forRoot({})],
providers: [
MicrosoftGetMessagesService,
MicrosoftHandleErrorService,
@ -29,6 +28,10 @@ describe('Microsoft get messages service', () => {
MicrosoftOAuth2ClientManagerService,
MicrosoftFetchByBatchService,
ConfigService,
{
provide: TwentyConfigService,
useValue: {},
},
],
}).compile();
@ -37,6 +40,10 @@ describe('Microsoft get messages service', () => {
);
});
afterEach(() => {
jest.clearAllMocks();
});
it('Should be defined', () => {
expect(service).toBeDefined();
});

View File

@ -10561,6 +10561,20 @@ __metadata:
languageName: node
linkType: hard
"@nestjs/schedule@npm:^3.0.0":
version: 3.0.4
resolution: "@nestjs/schedule@npm:3.0.4"
dependencies:
cron: "npm:2.4.3"
uuid: "npm:9.0.1"
peerDependencies:
"@nestjs/common": ^8.0.0 || ^9.0.0 || ^10.0.0
"@nestjs/core": ^8.0.0 || ^9.0.0 || ^10.0.0
reflect-metadata: ^0.1.12
checksum: 10c0/10af832f611139b586bd85714da1697276ddf51513c91cb20c82eb481c9b1741ec85560f7c55d6e560c8ab7f4665f95cb9fd77241a84f2b432b03e2819ba0f86
languageName: node
linkType: hard
"@nestjs/schematics@npm:^10.0.1":
version: 10.1.3
resolution: "@nestjs/schematics@npm:10.1.3"
@ -22593,6 +22607,13 @@ __metadata:
languageName: node
linkType: hard
"@types/luxon@npm:~3.3.0":
version: 3.3.8
resolution: "@types/luxon@npm:3.3.8"
checksum: 10c0/f2ffa31364eb94ca0474a196f533d301025a203bb2758ce0cf209f338cece0af169edea230b5c0b1a68a71adb02f369faa5ec0bd824deb8f0a08cac6803b1b06
languageName: node
linkType: hard
"@types/markdown-it@npm:12.2.3":
version: 12.2.3
resolution: "@types/markdown-it@npm:12.2.3"
@ -30201,6 +30222,16 @@ __metadata:
languageName: node
linkType: hard
"cron@npm:2.4.3":
version: 2.4.3
resolution: "cron@npm:2.4.3"
dependencies:
"@types/luxon": "npm:~3.3.0"
luxon: "npm:~3.3.0"
checksum: 10c0/3112d4cb0aa1c1129c0bb742eec205e38948806c907e21a0680d1aa83a1270bfade9fcf090c1604529684d21a64d74eb89075e25c16d48d95cf3c5b5d032f316
languageName: node
linkType: hard
"cross-env@npm:^7.0.3":
version: 7.0.3
resolution: "cross-env@npm:7.0.3"
@ -42038,6 +42069,13 @@ __metadata:
languageName: node
linkType: hard
"luxon@npm:~3.3.0":
version: 3.3.0
resolution: "luxon@npm:3.3.0"
checksum: 10c0/47f8e1e96b25441c799b8aa833b3f007fb1854713bcffc8c3384eda8e61fc9af1f038474d137274d2d386492f341c8a8c992fc78c213adfb3143780feba2776c
languageName: node
linkType: hard
"lz-string@npm:^1.4.4, lz-string@npm:^1.5.0":
version: 1.5.0
resolution: "lz-string@npm:1.5.0"
@ -55047,6 +55085,7 @@ __metadata:
"@nestjs/cli": "npm:10.3.0"
"@nestjs/devtools-integration": "npm:^0.1.6"
"@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch"
"@nestjs/schedule": "npm:^3.0.0"
"@node-saml/passport-saml": "npm:^5.0.0"
"@nx/js": "npm:18.3.3"
"@opentelemetry/api": "npm:^1.9.0"