From 759eb3070e538a53cc1fa76720793442e3c4994c Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Wed, 18 Sep 2024 17:35:08 +0200 Subject: [PATCH] Improve demo seed (#7125) We have a few issues on demo seeding: - redis metdata cache was not flushed - server ram graphql schema cache was not cleared on metadata version increment --- packages/twenty-server/package.json | 2 +- ...patch => @graphql-yoga+nestjs+2.1.0.patch} | 64 +++++++++++++------ .../data-seed-demo-workspace.service.ts | 8 +++ yarn.lock | 8 +-- 4 files changed, 59 insertions(+), 23 deletions(-) rename packages/twenty-server/patches/{@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch => @graphql-yoga+nestjs+2.1.0.patch} (92%) diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 958f9139b..c01f49991 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@esbuild-plugins/node-modules-polyfill": "^0.2.2", - "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch", + "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch", "@langchain/mistralai": "^0.0.24", "@langchain/openai": "^0.1.3", "@monaco-editor/react": "^4.6.0", diff --git a/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch b/packages/twenty-server/patches/@graphql-yoga+nestjs+2.1.0.patch similarity index 92% rename from packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch rename to packages/twenty-server/patches/@graphql-yoga+nestjs+2.1.0.patch index b15363c16..07d271f6e 100644 --- a/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch +++ b/packages/twenty-server/patches/@graphql-yoga+nestjs+2.1.0.patch @@ -1,5 +1,5 @@ diff --git a/dist/cjs/index.js b/dist/cjs/index.js -index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd38790b508 100644 +index 1684394..32602b3 100644 --- a/dist/cjs/index.js +++ b/dist/cjs/index.js @@ -3,10 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true }); @@ -26,7 +26,7 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 const app = this.httpAdapterHost.httpAdapter.getInstance(); preStartHook?.(app); // nest's logger doesnt have the info method -@@ -42,6 +46,40 @@ class AbstractYogaDriver extends graphql_2.AbstractGraphQLDriver { +@@ -42,6 +46,46 @@ class AbstractYogaDriver extends graphql_2.AbstractGraphQLDriver { } const yoga = (0, graphql_yoga_1.createYoga)({ ...options, @@ -60,6 +60,12 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 + schemas, + }); + ++ for (const key of this.schemaCache.keys()) { ++ if (key.startsWith(`${workspaceId}-`)) { ++ this.schemaCache.delete(key); ++ } ++ } ++ + this.schemaCache.set(cacheKey, mergedSchemas) + + return mergedSchemas; @@ -67,7 +73,7 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 graphqlEndpoint: options.path, // disable logging by default // however, if `true` use nest logger -@@ -54,11 +91,45 @@ class AbstractYogaDriver extends graphql_2.AbstractGraphQLDriver { +@@ -54,11 +98,51 @@ class AbstractYogaDriver extends graphql_2.AbstractGraphQLDriver { this.yoga = yoga; app.use(yoga.graphqlEndpoint, (req, res) => yoga(req, res, { req, res })); } @@ -107,6 +113,12 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 + schemas, + }); + ++ for (const key of this.schemaCache.keys()) { ++ if (key.startsWith(`${workspaceId}-`)) { ++ this.schemaCache.delete(key); ++ } ++ } ++ + this.schemaCache.set(cacheKey, mergedSchemas) + + return mergedSchemas; @@ -115,15 +127,19 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 // disable logging by default // however, if `true` use fastify logger diff --git a/dist/esm/index.js b/dist/esm/index.js -index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b39f580570 100644 +index 7068c51..b8cbf9e 100644 --- a/dist/esm/index.js +++ b/dist/esm/index.js -@@ -2,8 +2,12 @@ import { __decorate } from "tslib"; - import { printSchema } from 'graphql'; - import { createYoga, filter, pipe } from 'graphql-yoga'; - import { Injectable, Logger } from '@nestjs/common'; +@@ -1,9 +1,13 @@ +-import { __decorate } from "tslib"; +-import { printSchema } from 'graphql'; +-import { createYoga, filter, pipe } from 'graphql-yoga'; +import { mergeSchemas } from '@graphql-tools/schema'; + import { Injectable, Logger } from '@nestjs/common'; import { AbstractGraphQLDriver, GqlSubscriptionService, } from '@nestjs/graphql'; ++import { printSchema } from 'graphql'; ++import { createYoga, filter, pipe } from 'graphql-yoga'; ++import { __decorate } from "tslib"; export class AbstractYogaDriver extends AbstractGraphQLDriver { + + schemaCache = new Map(); @@ -140,7 +156,7 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 const app = this.httpAdapterHost.httpAdapter.getInstance(); preStartHook?.(app); // nest's logger doesnt have the info method -@@ -39,6 +43,40 @@ export class AbstractYogaDriver extends AbstractGraphQLDriver { +@@ -39,6 +43,46 @@ export class AbstractYogaDriver extends AbstractGraphQLDriver { } const yoga = createYoga({ ...options, @@ -171,9 +187,15 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 + } + + const mergedSchemas = mergeSchemas({ -+ schemas, ++ schemas, + }); + ++ for (const key of this.schemaCache.keys()) { ++ if (key.startsWith(`${workspaceId}-`)) { ++ this.schemaCache.delete(key); ++ } ++ } ++ + this.schemaCache.set(cacheKey, mergedSchemas) + + return mergedSchemas; @@ -181,7 +203,7 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 graphqlEndpoint: options.path, // disable logging by default // however, if `true` use nest logger -@@ -51,11 +88,45 @@ export class AbstractYogaDriver extends AbstractGraphQLDriver { +@@ -51,11 +95,51 @@ export class AbstractYogaDriver extends AbstractGraphQLDriver { this.yoga = yoga; app.use(yoga.graphqlEndpoint, (req, res) => yoga(req, res, { req, res })); } @@ -221,6 +243,12 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 + schemas, + }); + ++ for (const key of this.schemaCache.keys()) { ++ if (key.startsWith(`${workspaceId}-`)) { ++ this.schemaCache.delete(key); ++ } ++ } ++ + this.schemaCache.set(cacheKey, mergedSchemas) + + return mergedSchemas; @@ -229,7 +257,7 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 // disable logging by default // however, if `true` use fastify logger diff --git a/dist/typings/index.d.cts b/dist/typings/index.d.cts -index 2c6a9656193392680121487c7147db459d6b69ab..2f2b59f0e311f0526a7cfdad97372229301aabd7 100644 +index 2c6a965..2f2b59f 100644 --- a/dist/typings/index.d.cts +++ b/dist/typings/index.d.cts @@ -1,7 +1,8 @@ @@ -268,7 +296,7 @@ index 2c6a9656193392680121487c7147db459d6b69ab..2f2b59f0e311f0526a7cfdad97372229 }): void; subscriptionWithFilter(instanceRef: unknown, filterFn: (payload: TPayload, variables: TVariables, context: TContext) => boolean | Promise, createSubscribeContext: Function): (args_0: TPayload, args_1: TVariables, args_2: TContext) => Promise>; diff --git a/dist/typings/index.d.ts b/dist/typings/index.d.ts -index 2c6a9656193392680121487c7147db459d6b69ab..fd86daccf3e5a93ff44b568c9793c16d761f4f53 100644 +index 2c6a965..fd86dac 100644 --- a/dist/typings/index.d.ts +++ b/dist/typings/index.d.ts @@ -1,7 +1,8 @@ @@ -306,7 +334,7 @@ index 2c6a9656193392680121487c7147db459d6b69ab..fd86daccf3e5a93ff44b568c9793c16d }): void; subscriptionWithFilter(instanceRef: unknown, filterFn: (payload: TPayload, variables: TVariables, context: TContext) => boolean | Promise, createSubscribeContext: Function): (args_0: TPayload, args_1: TVariables, args_2: TContext) => Promise>; diff --git a/src/index.ts b/src/index.ts -index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac941edabb 100644 +index ce142f6..10e17d2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,9 +1,10 @@ @@ -422,7 +450,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac graphqlEndpoint: options.path, // disable logging by default // however, if `true` use nest logger -@@ -105,8 +149,8 @@ export abstract class AbstractYogaDriver< +@@ -105,8 +150,8 @@ export abstract class AbstractYogaDriver< options.logging == null ? false : options.logging @@ -433,7 +461,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac }); this.yoga = yoga as YogaDriverServerInstance; -@@ -115,7 +159,7 @@ export abstract class AbstractYogaDriver< +@@ -115,7 +160,7 @@ export abstract class AbstractYogaDriver< } protected registerFastify( @@ -442,7 +470,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac { preStartHook }: { preStartHook?: (app: FastifyInstance) => void } = {}, ) { const app: FastifyInstance = this.httpAdapterHost.httpAdapter.getInstance(); -@@ -124,6 +168,40 @@ export abstract class AbstractYogaDriver< +@@ -124,6 +169,40 @@ export abstract class AbstractYogaDriver< const yoga = createYoga>({ ...options, @@ -483,7 +511,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac graphqlEndpoint: options.path, // disable logging by default // however, if `true` use fastify logger -@@ -191,8 +268,8 @@ export class YogaDriver< +@@ -191,8 +270,8 @@ export class YogaDriver< const config: SubscriptionConfig = options.subscriptions === true ? { diff --git a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts index 7f55575a0..a7457d6a7 100644 --- a/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts +++ b/packages/twenty-server/src/database/commands/data-seed-demo-workspace/services/data-seed-demo-workspace.service.ts @@ -8,6 +8,9 @@ import { seedCoreSchema, } from 'src/database/typeorm-seeds/core/demo'; import { rawDataSource } from 'src/database/typeorm/raw/raw.datasource'; +import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; +import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; +import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; @@ -19,6 +22,8 @@ export class DataSeedDemoWorkspaceService { private readonly workspaceManagerService: WorkspaceManagerService, @InjectRepository(Workspace, 'core') protected readonly workspaceRepository: Repository, + @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) + private readonly workspaceSchemaCache: CacheStorageService, ) {} async seedDemo(): Promise { @@ -32,6 +37,9 @@ export class DataSeedDemoWorkspaceService { 'Could not get DEMO_WORKSPACE_IDS. Please specify in .env', ); } + + await this.workspaceSchemaCache.flush(); + for (const workspaceId of demoWorkspaceIds) { const existingWorkspaces = await this.workspaceRepository.findBy({ id: workspaceId, diff --git a/yarn.lock b/yarn.lock index 02ba4f2a8..0c03216bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6291,16 +6291,16 @@ __metadata: languageName: node linkType: hard -"@graphql-yoga/nestjs@patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch::locator=twenty-server%40workspace%3Apackages%2Ftwenty-server": +"@graphql-yoga/nestjs@patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch::locator=twenty-server%40workspace%3Apackages%2Ftwenty-server": version: 2.1.0 - resolution: "@graphql-yoga/nestjs@patch:@graphql-yoga/nestjs@npm%3A2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch::version=2.1.0&hash=96a960&locator=twenty-server%40workspace%3Apackages%2Ftwenty-server" + resolution: "@graphql-yoga/nestjs@patch:@graphql-yoga/nestjs@npm%3A2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch::version=2.1.0&hash=ba139d&locator=twenty-server%40workspace%3Apackages%2Ftwenty-server" peerDependencies: "@nestjs/common": ^10.0.0 "@nestjs/core": ^10.0.0 "@nestjs/graphql": ^12.0.0 graphql: ^15.0.0 || ^16.0.0 graphql-yoga: ^4.0.4 - checksum: 10c0/d7a9f1ff65642cf64e942ef1caf2596b0b2866118bd95200fc2112902e71d4aeb00fe436d5d253f2415f51cf79c48417f3859fae8ed4beaf2ac7e52d49eef3a1 + checksum: 10c0/9ab6b702399658c120a2250d1b69df1c9094deadea0df8da2c610114ce1089a4e373586f8e967179025a567d8343731aaf971d3278e32778bdd5a840db7e45bd languageName: node linkType: hard @@ -46879,7 +46879,7 @@ __metadata: resolution: "twenty-server@workspace:packages/twenty-server" dependencies: "@esbuild-plugins/node-modules-polyfill": "npm:^0.2.2" - "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch" + "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch" "@langchain/mistralai": "npm:^0.0.24" "@langchain/openai": "npm:^0.1.3" "@monaco-editor/react": "npm:^4.6.0"