admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Environment variables in admin panel (read only) - backend (#9943)

Browse Source 
Backend for https://github.com/twentyhq/core-team-issues/issues/293

POC - https://github.com/twentyhq/twenty/pull/9903

---------

Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
This commit is contained in:
nitin
2025-02-03 21:05:43 +05:30
committed by GitHub
parent 49e4484937
commit c8af90dc01
25 changed files with 1827 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
packages/twenty-server/src/engine/core-modules/environment/constants/__tests__/environment-variables-group-position.spec.ts Normal file
View File

@ -0,0 +1,26 @@
import { ENVIRONMENT_VARIABLES_GROUP_POSITION } from 'src/engine/core-modules/environment/constants/environment-variables-group-position';
import { EnvironmentVariablesGroup } from 'src/engine/core-modules/environment/enums/environment-variables-group.enum';
describe('ENVIRONMENT_VARIABLES_GROUP_POSITION', () => {
it('should include all EnvironmentVariablesGroup enum values', () => {
const enumValues = Object.values(EnvironmentVariablesGroup);
const positionKeys = Object.keys(ENVIRONMENT_VARIABLES_GROUP_POSITION);
enumValues.forEach((enumValue) => {
expect(positionKeys).toContain(enumValue);
});
positionKeys.forEach((key) => {
expect(enumValues).toContain(key);
});
expect(enumValues.length).toBe(positionKeys.length);
});
it('should have unique position values', () => {
const positions = Object.values(ENVIRONMENT_VARIABLES_GROUP_POSITION);
const uniquePositions = new Set(positions);
expect(positions.length).toBe(uniquePositions.size);
});
});

23
packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-group-position.ts Normal file
View File

@ -0,0 +1,23 @@
import { EnvironmentVariablesGroup } from 'src/engine/core-modules/environment/enums/environment-variables-group.enum';
export const ENVIRONMENT_VARIABLES_GROUP_POSITION: Record<
EnvironmentVariablesGroup,
number
> = {
[EnvironmentVariablesGroup.ServerConfig]: 100,
[EnvironmentVariablesGroup.Database]: 200,
[EnvironmentVariablesGroup.Security]: 300,
[EnvironmentVariablesGroup.Authentication]: 400,
[EnvironmentVariablesGroup.Cache]: 500,
[EnvironmentVariablesGroup.QueueConfig]: 600,
[EnvironmentVariablesGroup.Storage]: 700,
[EnvironmentVariablesGroup.Email]: 800,
[EnvironmentVariablesGroup.Frontend]: 900,
[EnvironmentVariablesGroup.Workspace]: 1000,
[EnvironmentVariablesGroup.Analytics]: 1100,
[EnvironmentVariablesGroup.Logging]: 1200,
[EnvironmentVariablesGroup.Billing]: 1300,
[EnvironmentVariablesGroup.Support]: 1400,
[EnvironmentVariablesGroup.LLM]: 1500,
[EnvironmentVariablesGroup.Serverless]: 1600,
};

4
packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-hidden-groups.ts Normal file
View File

@ -0,0 +1,4 @@
import { EnvironmentVariablesGroup } from 'src/engine/core-modules/environment/enums/environment-variables-group.enum';
export const ENVIRONMENT_VARIABLES_HIDDEN_GROUPS: Set<EnvironmentVariablesGroup> =
new Set([EnvironmentVariablesGroup.LLM]);

29
packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-masking-config.ts Normal file
View File

@ -0,0 +1,29 @@
import { EnvironmentVariablesMaskingStrategies } from 'src/engine/core-modules/environment/enums/environment-variables-masking-strategies.enum';
type LastNCharsConfig = {
strategy: EnvironmentVariablesMaskingStrategies.LAST_N_CHARS;
chars: number;
};
type HidePasswordConfig = {
strategy: EnvironmentVariablesMaskingStrategies.HIDE_PASSWORD;
};
type MaskingConfigType = {
APP_SECRET: LastNCharsConfig;
PG_DATABASE_URL: HidePasswordConfig;
REDIS_URL: HidePasswordConfig;
};
export const ENVIRONMENT_VARIABLES_MASKING_CONFIG: MaskingConfigType = {
APP_SECRET: {
strategy: EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
chars: 5,
},
PG_DATABASE_URL: {
strategy: EnvironmentVariablesMaskingStrategies.HIDE_PASSWORD,
},
REDIS_URL: {
strategy: EnvironmentVariablesMaskingStrategies.HIDE_PASSWORD,
},
} as const;

2
packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-metadata-decorator-key.ts Normal file
View File

@ -0,0 +1,2 @@
export const ENVIRONMENT_VARIABLES_METADATA_DECORATOR_KEY =
'environment-variables-metadata' as const;

2
packages/twenty-server/src/engine/core-modules/environment/constants/environment-variables-metadata-decorator-names-key.ts Normal file
View File

@ -0,0 +1,2 @@
export const ENVIRONMENT_VARIABLES_METADATA_DECORATOR_NAMES_KEY =
'environment-variable-names' as const;

52
packages/twenty-server/src/engine/core-modules/environment/decorators/environment-variables-metadata.decorator.ts Normal file
View File

@ -0,0 +1,52 @@
import { registerDecorator, ValidationOptions } from 'class-validator';
import { EnvironmentVariablesGroup } from 'src/engine/core-modules/environment/enums/environment-variables-group.enum';
import { EnvironmentVariablesSubGroup } from 'src/engine/core-modules/environment/enums/environment-variables-sub-group.enum';
import { TypedReflect } from 'src/utils/typed-reflect';
export interface EnvironmentVariablesMetadataOptions {
group: EnvironmentVariablesGroup;
subGroup?: EnvironmentVariablesSubGroup;
description: string;
sensitive?: boolean;
}
export type EnvironmentVariablesMetadataMap = {
[key: string]: EnvironmentVariablesMetadataOptions;
};
export function EnvironmentVariablesMetadata(
options: EnvironmentVariablesMetadataOptions,
validationOptions?: ValidationOptions,
): PropertyDecorator {
return (target: object, propertyKey: string | symbol) => {
const existingMetadata: EnvironmentVariablesMetadataMap =
TypedReflect.getMetadata('environment-variables', target.constructor) ??
{};
TypedReflect.defineMetadata(
'environment-variables',
{
...existingMetadata,
[propertyKey.toString()]: options,
},
target.constructor,
);
registerDecorator({
name: propertyKey.toString(),
target: target.constructor,
propertyName: propertyKey.toString(),
options: validationOptions,
constraints: [options],
validator: {
validate() {
return true;
},
defaultMessage() {
return `${propertyKey.toString()} has invalid metadata`;
},
},
});
};
}

18
packages/twenty-server/src/engine/core-modules/environment/enums/environment-variables-group.enum.ts Normal file
View File

@ -0,0 +1,18 @@
export enum EnvironmentVariablesGroup {
Authentication = 'authentication',
Email = 'email',
Database = 'database',
Storage = 'storage',
ServerConfig = 'server-config',
QueueConfig = 'queue-config',
Logging = 'logging',
Cache = 'cache',
Analytics = 'analytics',
Billing = 'billing',
Frontend = 'frontend',
Security = 'security',
Serverless = 'serverless',
Support = 'support',
LLM = 'llm',
Workspace = 'workspace',
}

4
packages/twenty-server/src/engine/core-modules/environment/enums/environment-variables-masking-strategies.enum.ts Normal file
View File

@ -0,0 +1,4 @@
export enum EnvironmentVariablesMaskingStrategies {
LAST_N_CHARS = 'LAST_N_CHARS',
HIDE_PASSWORD = 'HIDE_PASSWORD',
}

17
packages/twenty-server/src/engine/core-modules/environment/enums/environment-variables-sub-group.enum.ts Normal file
View File

@ -0,0 +1,17 @@
export enum EnvironmentVariablesSubGroup {
PasswordAuth = 'password-auth',
GoogleAuth = 'google-auth',
MicrosoftAuth = 'microsoft-auth',
SmtpConfig = 'smtp-config',
EmailSettings = 'email-settings',
S3Config = 's3-config',
Tokens = 'tokens',
SSL = 'ssl',
RateLimiting = 'rate-limiting',
LambdaConfig = 'lambda-config',
TinybirdConfig = 'tinybird-config',
StripeConfig = 'stripe-config',
SentryConfig = 'sentry-config',
FrontSupportConfig = 'front-support-config',
CloudflareConfig = 'cloudflare-config',
}

1273
packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts
View File

File diff suppressed because it is too large Load Diff

89
packages/twenty-server/src/engine/core-modules/environment/environment.service.spec.ts
View File

@ -1,10 +1,12 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ConfigService } from '@nestjs/config';
import { Test, TestingModule } from '@nestjs/testing';
import { EnvironmentVariables } from 'src/engine/core-modules/environment/environment-variables';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
describe('EnvironmentService', () => {
let service: EnvironmentService;
let configService: ConfigService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
@ -12,15 +14,98 @@ describe('EnvironmentService', () => {
EnvironmentService,
{
provide: ConfigService,
useValue: {},
useValue: {
get: jest.fn(),
},
},
],
}).compile();
service = module.get<EnvironmentService>(EnvironmentService);
configService = module.get<ConfigService>(ConfigService);
Reflect.defineMetadata('environment-variables', {}, EnvironmentVariables);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('getAll()', () => {
it('should return empty object when no environment variables are defined', () => {
const result = service.getAll();
expect(result).toEqual({});
});
it('should return environment variables with their metadata', () => {
const mockMetadata = {
TEST_VAR: {
title: 'Test Var',
description: 'Test Description',
},
};
Reflect.defineMetadata(
'environment-variables',
mockMetadata,
EnvironmentVariables,
);
jest.spyOn(configService, 'get').mockReturnValue('test-value');
const result = service.getAll();
expect(result).toEqual({
TEST_VAR: {
value: 'test-value',
metadata: mockMetadata.TEST_VAR,
},
});
});
it('should mask sensitive data according to masking config', () => {
const mockMetadata = {
APP_SECRET: {
title: 'App Secret',
description: 'Application secret key',
sensitive: true,
},
};
Reflect.defineMetadata(
'environment-variables',
mockMetadata,
EnvironmentVariables,
);
jest.spyOn(configService, 'get').mockReturnValue('super-secret-value');
const result = service.getAll();
expect(result.APP_SECRET.value).not.toBe('super-secret-value');
expect(result.APP_SECRET.value).toMatch(/^\*+[a-zA-Z0-9]{5}$/);
});
it('should use default value when environment variable is not set', () => {
const mockMetadata = {
DEBUG_PORT: {
title: 'Debug Port',
description: 'Debug port number',
},
};
Reflect.defineMetadata(
'environment-variables',
mockMetadata,
EnvironmentVariables,
);
jest.spyOn(configService, 'get').mockReturnValue(undefined);
const result = service.getAll();
expect(result.DEBUG_PORT.value).toBe(9000);
});
});
});

61
packages/twenty-server/src/engine/core-modules/environment/environment.service.ts
View File

@ -2,7 +2,12 @@
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { ENVIRONMENT_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/environment/constants/environment-variables-masking-config';
import { EnvironmentVariablesMetadataOptions } from 'src/engine/core-modules/environment/decorators/environment-variables-metadata.decorator';
import { EnvironmentVariablesMaskingStrategies } from 'src/engine/core-modules/environment/enums/environment-variables-masking-strategies.enum';
import { EnvironmentVariables } from 'src/engine/core-modules/environment/environment-variables';
import { environmentVariableMaskSensitiveData } from 'src/engine/core-modules/environment/utils/environment-variable-mask-sensitive-data.util';
import { TypedReflect } from 'src/utils/typed-reflect';
@Injectable()
export class EnvironmentService {
@ -14,4 +19,60 @@ export class EnvironmentService {
new EnvironmentVariables()[key],
);
}
getAll(): Record<
string,
{
value: EnvironmentVariables[keyof EnvironmentVariables];
metadata: EnvironmentVariablesMetadataOptions;
}
> {
const result: Record<
string,
{
value: EnvironmentVariables[keyof EnvironmentVariables];
metadata: EnvironmentVariablesMetadataOptions;
}
> = {};
const envVars = new EnvironmentVariables();
const metadata =
TypedReflect.getMetadata('environment-variables', EnvironmentVariables) ??
{};
Object.entries(metadata).forEach(([key, envMetadata]) => {
let value =
this.configService.get(key) ??
envVars[key as keyof EnvironmentVariables] ??
'';
if (
typeof value === 'string' &&
key in ENVIRONMENT_VARIABLES_MASKING_CONFIG
) {
const varMaskingConfig =
ENVIRONMENT_VARIABLES_MASKING_CONFIG[
key as keyof typeof ENVIRONMENT_VARIABLES_MASKING_CONFIG
];
const options =
varMaskingConfig.strategy ===
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS
? { chars: varMaskingConfig.chars }
: undefined;
value = environmentVariableMaskSensitiveData(
value,
varMaskingConfig.strategy,
{ ...options, variableName: key },
);
}
result[key] = {
value,
metadata: envMetadata,
};
});
return result;
}
}

105
packages/twenty-server/src/engine/core-modules/environment/utils/__tests__/environment-variable-mask-sensitive-data.util.spec.ts Normal file
View File

@ -0,0 +1,105 @@
import { EnvironmentVariablesMaskingStrategies } from 'src/engine/core-modules/environment/enums/environment-variables-masking-strategies.enum';
import { environmentVariableMaskSensitiveData } from 'src/engine/core-modules/environment/utils/environment-variable-mask-sensitive-data.util';
describe('environmentVariableMaskSensitiveData', () => {
describe('LAST_N_CHARS strategy', () => {
it('should mask all but the last 5 characters by default', () => {
const result = environmentVariableMaskSensitiveData(
'mysecretvalue123',
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
);
expect(result).toBe('********ue123');
});
it('should mask all but the specified number of characters', () => {
const result = environmentVariableMaskSensitiveData(
'mysecretvalue123',
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
{ chars: 3 },
);
expect(result).toBe('********123');
});
it('should return all asterisks if value is shorter than mask length', () => {
const result = environmentVariableMaskSensitiveData(
'123',
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
{ chars: 5 },
);
expect(result).toBe('********');
});
it('should handle empty string', () => {
const result = environmentVariableMaskSensitiveData(
'',
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
);
expect(result).toBe('');
});
});
describe('HIDE_PASSWORD strategy', () => {
it('should mask password in URL', () => {
const result = environmentVariableMaskSensitiveData(
'postgresql://user:password123@localhost:5432/db',
EnvironmentVariablesMaskingStrategies.HIDE_PASSWORD,
);
expect(result).toBe('postgresql://********:********@localhost:5432/db');
});
it('should handle URL without password', () => {
const result = environmentVariableMaskSensitiveData(
'postgresql://localhost:5432/db',
EnvironmentVariablesMaskingStrategies.HIDE_PASSWORD,
);
expect(result).toBe('postgresql://localhost:5432/db');
});
it('should throw error for invalid URLs', () => {
expect(() =>
environmentVariableMaskSensitiveData(
'not-a-url',
EnvironmentVariablesMaskingStrategies.HIDE_PASSWORD,
{ variableName: 'TEST_URL' },
),
).toThrow(
'Invalid URL format for TEST_URL in HIDE_PASSWORD masking strategy',
);
});
});
describe('edge cases', () => {
it('should handle null value', () => {
const result = environmentVariableMaskSensitiveData(
null as any,
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
);
expect(result).toBeNull();
});
it('should handle undefined value', () => {
const result = environmentVariableMaskSensitiveData(
undefined as any,
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
);
expect(result).toBeUndefined();
});
it('should handle non-string value', () => {
const result = environmentVariableMaskSensitiveData(
123 as any,
EnvironmentVariablesMaskingStrategies.LAST_N_CHARS,
);
expect(result).toBe(123);
});
});
});

38
packages/twenty-server/src/engine/core-modules/environment/utils/environment-variable-mask-sensitive-data.util.ts Normal file
View File

@ -0,0 +1,38 @@
import { EnvironmentVariablesMaskingStrategies } from 'src/engine/core-modules/environment/enums/environment-variables-masking-strategies.enum';
export const environmentVariableMaskSensitiveData = (
value: string,
strategy: EnvironmentVariablesMaskingStrategies,
options?: { chars?: number; variableName?: string },
): string => {
if (!value || typeof value !== 'string') return value;
switch (strategy) {
case EnvironmentVariablesMaskingStrategies.LAST_N_CHARS: {
const n = Math.max(1, options?.chars ?? 5);
return value.length > n ? `********${value.slice(-n)}` : '********';
}
case EnvironmentVariablesMaskingStrategies.HIDE_PASSWORD: {
try {
const url = new URL(value);
if (url.password) {
url.password = '********';
}
if (url.username) {
url.username = '********';
}
return url.toString();
} catch {
throw new Error(
`Invalid URL format for ${options?.variableName || 'environment variable'} in HIDE_PASSWORD masking strategy`,
);
}
}
default:
return value;
}
};