From ba2ee0da723e23601dc7cc9e151261ee1567ace1 Mon Sep 17 00:00:00 2001 From: Thomas Mol Date: Wed, 16 Oct 2024 17:17:44 +0200 Subject: [PATCH] Refactor Redis connections to use Redis URL - closes #7421 (#7736) Closes #7421 This pull request consolidates Redis connection parameters into a single `REDIS_URL` environment variable across various configuration files and code modules. The most important changes include updates to environment variable files, Docker and Kubernetes configurations, and code modules to utilize the new `REDIS_URL` format. ### Environment Variable Updates: * [`packages/twenty-docker/.env.example`](diffhunk://#diff-f4b5e7acc0dde630eafb2228390ca56bd56db0d183582be6433a9ee808088a4cL8-R8): Replaced `REDIS_HOST` and `REDIS_PORT` with `REDIS_URL`. * [`packages/twenty-server/.env.example`](diffhunk://#diff-c06e244338b33286ea90221635809785352a971de53f647ea65650124ae74ad2L53-R53): Replaced `REDIS_HOST`, `REDIS_PORT`, `REDIS_USERNAME`, and `REDIS_PASSWORD` with `REDIS_URL`. * [`packages/twenty-server/.env.test`](diffhunk://#diff-def44a08e699c9deca2f72c9f87951de5d33d9ccf5621eab2f34978df8ad0954L16-R16): Replaced `REDIS_HOST`, `REDIS_PORT`, `REDIS_USERNAME`, and `REDIS_PASSWORD` with `REDIS_URL`. ### Docker and Kubernetes Configuration Updates: * [`packages/twenty-docker/docker-compose.yml`](diffhunk://#diff-545fb51ad66f93f727bb8f8b375dc6858b33348d91d5e51ad931fb0bbc1affeeL28-R28): Replaced `REDIS_HOST` and `REDIS_PORT` with `REDIS_URL` in multiple service definitions. [[1]](diffhunk://#diff-545fb51ad66f93f727bb8f8b375dc6858b33348d91d5e51ad931fb0bbc1affeeL28-R28) [[2]](diffhunk://#diff-545fb51ad66f93f727bb8f8b375dc6858b33348d91d5e51ad931fb0bbc1affeeL62-R61) * [`packages/twenty-docker/k8s/manifests/deployment-server.yaml`](diffhunk://#diff-91623ed4e8b2088947cfa9a5dad76b6013e8db0c150d84347a215fa9ad78bf04L44-R45): Replaced `REDIS_HOST` and `REDIS_PORT` with `REDIS_URL`. * [`packages/twenty-docker/k8s/manifests/deployment-worker.yaml`](diffhunk://#diff-8532debd131ce168a2527fa6a9be6405792178576ee47e2eef0cd9f3ff4a8f8cL43-R44): Replaced `REDIS_HOST` and `REDIS_PORT` with `REDIS_URL`. * [`packages/twenty-docker/k8s/terraform/deployment-server.tf`](diffhunk://#diff-b4c468660ab00bd03589d0d47502c477ca83f7d876ff196534880b02ba46dce0L64-R65): Replaced `REDIS_HOST` and `REDIS_PORT` with `REDIS_URL`. * [`packages/twenty-docker/k8s/terraform/deployment-worker.tf`](diffhunk://#diff-e5d6a1f68d5391e2120aef6261f22f905aa1ce0a567e31b21a27d44f313eaecaL62-R63): Replaced `REDIS_HOST` and `REDIS_PORT` with `REDIS_URL`. ### Code Module Updates: * [`packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts`](diffhunk://#diff-06e787a7c8a48022d5909b5df9b8c6ca192521cf32f51d7f561cee937bed6678L23-R35): Replaced `REDIS_HOST` and `REDIS_PORT` with `REDIS_URL`. * [`packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts`](diffhunk://#diff-26ce615693b053eda02aa48aa2e30400381a2588dcb08d4a9dc3b0bf5bdd6fe7L378-R384): Added validation for `REDIS_URL`. * [`packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts`](diffhunk://#diff-7bd644d28bdd86c159c1d71242753df5d5acd91c73c6e60b2e35caa53ed3836bR6): Replaced `REDIS_HOST`, `REDIS_PORT`, `REDIS_USERNAME`, and `REDIS_PASSWORD` with `REDIS_URL`. [[1]](diffhunk://#diff-7bd644d28bdd86c159c1d71242753df5d5acd91c73c6e60b2e35caa53ed3836bR6) [[2]](diffhunk://#diff-7bd644d28bdd86c159c1d71242753df5d5acd91c73c6e60b2e35caa53ed3836bL35-R41) ### Documentation Updates: * [`packages/twenty-website/src/content/developers/self-hosting/self-hosting-var.mdx`](diffhunk://#diff-c4cc78a3ce18b6edb10f1aee8990271e1d2796a8c06c1c6ae3b68db8d52278a3L37-R37): Updated documentation to reflect the change to `REDIS_URL`. * [`packages/twenty-website/src/content/developers/self-hosting/upgrade-guide.mdx`](diffhunk://#diff-c7d757829f6128e1f47aa1955bde561292dce558280511fab66281afa99042a5R102-R112): Added upgrade instructions for the new `REDIS_URL` variable. --------- Co-authored-by: Weiko Co-authored-by: Charles Bochet --- packages/twenty-docker/.env.example | 3 +- packages/twenty-docker/docker-compose.yml | 6 ++-- .../k8s/manifests/deployment-server.yaml | 6 ++-- .../k8s/manifests/deployment-worker.yaml | 6 ++-- .../k8s/terraform/deployment-server.tf | 8 ++--- .../k8s/terraform/deployment-worker.tf | 9 ++---- packages/twenty-server/.env.example | 5 +--- packages/twenty-server/.env.test | 5 +--- .../cache-storage.module-factory.ts | 17 +++-------- .../environment/environment-variables.ts | 20 ++++++++----- .../message-queue/drivers/bullmq.driver.ts | 4 +-- .../message-queue.module-factory.ts | 29 ++++++++++--------- .../self-hosting/self-hosting-var.mdx | 5 +--- .../developers/self-hosting/upgrade-guide.mdx | 11 +++++++ 14 files changed, 59 insertions(+), 75 deletions(-) diff --git a/packages/twenty-docker/.env.example b/packages/twenty-docker/.env.example index 59d8d03f9..c1a7a9d3b 100644 --- a/packages/twenty-docker/.env.example +++ b/packages/twenty-docker/.env.example @@ -3,10 +3,9 @@ TAG=latest # POSTGRES_ADMIN_PASSWORD=replace_me_with_a_strong_password PG_DATABASE_HOST=db:5432 +REDIS_URL=redis://redis:6379 SERVER_URL=http://localhost:3000 -# REDIS_HOST=redis -# REDIS_PORT=6379 # Use openssl rand -base64 32 for each secret # ACCESS_TOKEN_SECRET=replace_me_with_a_random_string_access diff --git a/packages/twenty-docker/docker-compose.yml b/packages/twenty-docker/docker-compose.yml index b2efc1a16..8800f4f3f 100644 --- a/packages/twenty-docker/docker-compose.yml +++ b/packages/twenty-docker/docker-compose.yml @@ -25,8 +25,7 @@ services: PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default SERVER_URL: ${SERVER_URL} FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL} - REDIS_PORT: ${REDIS_PORT:-6379} - REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_URL: ${REDIS_URL:-redis://localhost:6379} ENABLE_DB_MIGRATIONS: "true" @@ -59,8 +58,7 @@ services: PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default SERVER_URL: ${SERVER_URL} FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL} - REDIS_PORT: ${REDIS_PORT:-6379} - REDIS_HOST: ${REDIS_HOST:-redis} + REDIS_URL: ${REDIS_URL:-redis://localhost:6379} ENABLE_DB_MIGRATIONS: "false" # it already runs on the server diff --git a/packages/twenty-docker/k8s/manifests/deployment-server.yaml b/packages/twenty-docker/k8s/manifests/deployment-server.yaml index b1229d649..99e5c6013 100644 --- a/packages/twenty-docker/k8s/manifests/deployment-server.yaml +++ b/packages/twenty-docker/k8s/manifests/deployment-server.yaml @@ -41,10 +41,8 @@ spec: value: "https://crm.example.com:443" - name: "PG_DATABASE_URL" value: "postgres://twenty:twenty@twenty-db.twentycrm.svc.cluster.local/default" - - name: "REDIS_HOST" - value: "twentycrm-redis.twentycrm.svc.cluster.local" - - name: "REDIS_PORT" - value: 6379 + - name: "REDIS_URL" + value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379" - name: ENABLE_DB_MIGRATIONS value: "true" - name: SIGN_IN_PREFILLED diff --git a/packages/twenty-docker/k8s/manifests/deployment-worker.yaml b/packages/twenty-docker/k8s/manifests/deployment-worker.yaml index b3a7e07a1..92d0322e5 100644 --- a/packages/twenty-docker/k8s/manifests/deployment-worker.yaml +++ b/packages/twenty-docker/k8s/manifests/deployment-worker.yaml @@ -40,10 +40,8 @@ spec: value: "bull-mq" - name: "CACHE_STORAGE_TYPE" value: "redis" - - name: "REDIS_HOST" - value: "twentycrm-redis.twentycrm.svc.cluster.local" - - name: "REDIS_PORT" - value: 6379 + - name: "REDIS_URL" + value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379" - name: ACCESS_TOKEN_SECRET valueFrom: secretKeyRef: diff --git a/packages/twenty-docker/k8s/terraform/deployment-server.tf b/packages/twenty-docker/k8s/terraform/deployment-server.tf index 1868b1762..0f643f5c6 100644 --- a/packages/twenty-docker/k8s/terraform/deployment-server.tf +++ b/packages/twenty-docker/k8s/terraform/deployment-server.tf @@ -61,12 +61,8 @@ resource "kubernetes_deployment" "twentycrm_server" { value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${kubernetes_service.twentycrm_db.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default" } env { - name = "REDIS_HOST" - value = "${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local" - } - env { - name = "REDIS_PORT" - value = 6379 + name = "REDIS_URL" + value = "redis://${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local:6379" } env { name = "ENABLE_DB_MIGRATIONS" diff --git a/packages/twenty-docker/k8s/terraform/deployment-worker.tf b/packages/twenty-docker/k8s/terraform/deployment-worker.tf index 78e5ea6dc..163f02c49 100644 --- a/packages/twenty-docker/k8s/terraform/deployment-worker.tf +++ b/packages/twenty-docker/k8s/terraform/deployment-worker.tf @@ -59,13 +59,8 @@ resource "kubernetes_deployment" "twentycrm_worker" { } env { - name = "REDIS_HOST" - value = "${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local" - } - - env { - name = "REDIS_PORT" - value = 6379 + name = "REDIS_URL" + value = "redis://${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local:6379" } env { diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index 6f8167d6b..f43d3ed7b 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -50,10 +50,7 @@ SIGN_IN_PREFILLED=true # SENTRY_FRONT_DSN=https://xxx@xxx.ingest.sentry.io/xxx # LOG_LEVELS=error,warn # MESSAGE_QUEUE_TYPE=pg-boss -# REDIS_HOST=127.0.0.1 -# REDIS_PORT=6379 -# REDIS_USERNAME= -# REDIS_PASSWORD= +# REDIS_URL=redis://localhost:6379 # DEMO_WORKSPACE_IDS=REPLACE_ME_WITH_A_RANDOM_UUID # SERVER_URL=http://localhost:3000 # WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION=30 diff --git a/packages/twenty-server/.env.test b/packages/twenty-server/.env.test index ed0c63d78..b8fb82f4c 100644 --- a/packages/twenty-server/.env.test +++ b/packages/twenty-server/.env.test @@ -13,10 +13,7 @@ DEMO_WORKSPACE_IDS=63db4589-590f-42b3-bdf1-85268b3da02f,8de58f3f-7e86-4a0b-998d- MUTATION_MAXIMUM_RECORD_AFFECTED=100 MESSAGE_QUEUE_TYPE=pg-boss CACHE_STORAGE_TYPE=redis -REDIS_HOST=127.0.0.1 -REDIS_PORT=6379 -REDIS_USERNAME=default -REDIS_PASSWORD= +REDIS_URL=redis://localhost:6379 AUTH_GOOGLE_ENABLED=false MESSAGING_PROVIDER_GMAIL_ENABLED=false diff --git a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts index ce7cb740a..5a197a98e 100644 --- a/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts +++ b/packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts @@ -20,27 +20,18 @@ export const cacheStorageModuleFactory = ( return cacheModuleOptions; } case CacheStorageType.Redis: { - const host = environmentService.get('REDIS_HOST'); - const port = environmentService.get('REDIS_PORT'); + const connectionString = environmentService.get('REDIS_URL'); - if (!(host && port)) { + if (!connectionString) { throw new Error( - `${cacheStorageType} cache storage requires host: ${host} and port: ${port} to be defined, check your .env file`, + `${cacheStorageType} cache storage requires REDIS_URL to be defined, check your .env file`, ); } - const username = environmentService.get('REDIS_USERNAME'); - const password = environmentService.get('REDIS_PASSWORD'); - return { ...cacheModuleOptions, store: redisStore, - socket: { - host, - port, - username, - password, - }, + url: connectionString, }; } default: diff --git a/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts b/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts index cb5b2fbe2..faab8c9d6 100644 --- a/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts @@ -375,14 +375,18 @@ export class EnvironmentVariables { @IsNumber() MUTATION_MAXIMUM_AFFECTED_RECORDS = 100; - REDIS_HOST = '127.0.0.1'; - - @CastToPositiveNumber() - REDIS_PORT = 6379; - - REDIS_USERNAME: string; - - REDIS_PASSWORD: string; + @IsOptional() + @ValidateIf( + (env) => + env.CACHE_STORAGE_TYPE === CacheStorageType.Redis || + env.MESSAGE_QUEUE_TYPE === MessageQueueDriverType.BullMQ, + ) + @IsUrl({ + protocols: ['redis'], + require_tld: false, + allow_underscores: true, + }) + REDIS_URL: string; API_TOKEN_EXPIRES_IN = '100y'; diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts index cf4c414c8..8525330b5 100644 --- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts @@ -1,15 +1,15 @@ import { OnModuleDestroy } from '@nestjs/common'; -import omitBy from 'lodash.omitby'; import { JobsOptions, Queue, QueueOptions, Worker } from 'bullmq'; +import omitBy from 'lodash.omitby'; import { QueueCronJobOptions, QueueJobOptions, } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface'; +import { MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface'; import { MessageQueueJob } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface'; import { MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface'; -import { MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts index ad18f6cd8..2948b2ad9 100644 --- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts +++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts @@ -1,7 +1,12 @@ +import { ConnectionOptions } from 'tls'; + import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; import { + BullMQDriverFactoryOptions, MessageQueueDriverType, MessageQueueModuleOptions, + PgBossDriverFactoryOptions, + SyncDriverFactoryOptions, } from 'src/engine/core-modules/message-queue/interfaces'; /** @@ -19,7 +24,7 @@ export const messageQueueModuleFactory = async ( return { type: MessageQueueDriverType.Sync, options: {}, - }; + } satisfies SyncDriverFactoryOptions; } case MessageQueueDriverType.PgBoss: { const connectionString = environmentService.get('PG_DATABASE_URL'); @@ -29,25 +34,23 @@ export const messageQueueModuleFactory = async ( options: { connectionString, }, - }; + } satisfies PgBossDriverFactoryOptions; } case MessageQueueDriverType.BullMQ: { - const host = environmentService.get('REDIS_HOST'); - const port = environmentService.get('REDIS_PORT'); - const username = environmentService.get('REDIS_USERNAME'); - const password = environmentService.get('REDIS_PASSWORD'); + const connectionString = environmentService.get('REDIS_URL'); + + if (!connectionString) { + throw new Error( + `${MessageQueueDriverType.BullMQ} message queue requires REDIS_URL to be defined, check your .env file`, + ); + } return { type: MessageQueueDriverType.BullMQ, options: { - connection: { - host, - port, - username, - password, - }, + connection: connectionString as ConnectionOptions, }, - }; + } satisfies BullMQDriverFactoryOptions; } default: throw new Error( diff --git a/packages/twenty-website/src/content/developers/self-hosting/self-hosting-var.mdx b/packages/twenty-website/src/content/developers/self-hosting/self-hosting-var.mdx index cda5fb3a3..d679096f8 100644 --- a/packages/twenty-website/src/content/developers/self-hosting/self-hosting-var.mdx +++ b/packages/twenty-website/src/content/developers/self-hosting/self-hosting-var.mdx @@ -34,10 +34,7 @@ yarn command:prod cron:calendar:calendar-event-list-fetch