diff --git a/server/package.json b/server/package.json index b7b75b519..bc4526ad3 100644 --- a/server/package.json +++ b/server/package.json @@ -41,7 +41,6 @@ "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.3.2", "@nestjs/core": "^9.0.0", - "@nestjs/event-emitter": "^2.0.3", "@nestjs/graphql": "^12.0.8", "@nestjs/jwt": "^10.0.3", "@nestjs/passport": "^9.0.3", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index fc5146c86..da2bc4e25 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -2,7 +2,6 @@ import { Module } from '@nestjs/common'; import { GraphQLModule } from '@nestjs/graphql'; import { ConfigModule } from '@nestjs/config'; import { APP_FILTER, ContextIdFactory, ModuleRef } from '@nestjs/core'; -import { EventEmitterModule } from '@nestjs/event-emitter'; import { YogaDriver, YogaDriverConfig } from '@graphql-yoga/nestjs'; import GraphQLJSON from 'graphql-type-json'; @@ -104,7 +103,6 @@ import { ExceptionFilter } from './filters/exception.filter'; resolvers: { JSON: GraphQLJSON }, plugins: [], }), - EventEmitterModule.forRoot(), HealthModule, IntegrationsModule, CoreModule, diff --git a/server/src/database/typeorm/metadata/migrations/1700650554672-addWorkspaceCacheVersion.ts b/server/src/database/typeorm/metadata/migrations/1700650554672-addWorkspaceCacheVersion.ts new file mode 100644 index 000000000..062fd1f4c --- /dev/null +++ b/server/src/database/typeorm/metadata/migrations/1700650554672-addWorkspaceCacheVersion.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddWorkspaceCacheVersion1700650554672 implements MigrationInterface { + name = 'AddWorkspaceCacheVersion1700650554672' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "metadata"."workspaceCacheVersion" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "workspaceId" character varying NOT NULL, "version" character varying NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_1a80ecf2638b477809403cc26ed" UNIQUE ("workspaceId"), CONSTRAINT "PK_5d502f8dbfb5b9a8bf2439320e9" PRIMARY KEY ("id"))`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "metadata"."workspaceCacheVersion"`); + } + +} diff --git a/server/src/metadata/workspace-cache-version/workspace-cache-version.entity.ts b/server/src/metadata/workspace-cache-version/workspace-cache-version.entity.ts new file mode 100644 index 000000000..682df8e90 --- /dev/null +++ b/server/src/metadata/workspace-cache-version/workspace-cache-version.entity.ts @@ -0,0 +1,25 @@ +import { + Column, + CreateDateColumn, + Entity, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm'; + +@Entity('workspaceCacheVersion') +export class WorkspaceCacheVersionEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ unique: true }) + workspaceId: string; + + @Column() + version: string; + + @CreateDateColumn() + createdAt: Date; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/server/src/metadata/workspace-cache-version/workspace-cache-version.module.ts b/server/src/metadata/workspace-cache-version/workspace-cache-version.module.ts new file mode 100644 index 000000000..3eaa886ec --- /dev/null +++ b/server/src/metadata/workspace-cache-version/workspace-cache-version.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { WorkspaceCacheVersionEntity } from 'src/metadata/workspace-cache-version/workspace-cache-version.entity'; +import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([WorkspaceCacheVersionEntity], 'metadata'), + ], + exports: [WorkspaceCacheVersionService], + providers: [WorkspaceCacheVersionService], +}) +export class WorkspaceCacheVersionModule {} diff --git a/server/src/metadata/workspace-cache-version/workspace-cache-version.service.ts b/server/src/metadata/workspace-cache-version/workspace-cache-version.service.ts new file mode 100644 index 000000000..7a347b7a0 --- /dev/null +++ b/server/src/metadata/workspace-cache-version/workspace-cache-version.service.ts @@ -0,0 +1,38 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { WorkspaceCacheVersionEntity } from 'src/metadata/workspace-cache-version/workspace-cache-version.entity'; + +@Injectable() +export class WorkspaceCacheVersionService { + constructor( + @InjectRepository(WorkspaceCacheVersionEntity, 'metadata') + private readonly workspaceCacheVersionRepository: Repository, + ) {} + + async incrementVersion(workspaceId: string): Promise { + const workspaceCacheVersion = + (await this.workspaceCacheVersionRepository.findOne({ + where: { workspaceId }, + })) ?? { version: '0' }; + + await this.workspaceCacheVersionRepository.upsert( + { + workspaceId, + version: `${+workspaceCacheVersion.version + 1}`, + }, + ['workspaceId'], + ); + } + + async getVersion(workspaceId: string): Promise { + const workspaceCacheVersion = + await this.workspaceCacheVersionRepository.findOne({ + where: { workspaceId }, + }); + + return workspaceCacheVersion?.version ?? '0'; + } +} diff --git a/server/src/metadata/workspace-migration/workspace-migration.entity.ts b/server/src/metadata/workspace-migration/workspace-migration.entity.ts index 798dc01aa..1e847c935 100644 --- a/server/src/metadata/workspace-migration/workspace-migration.entity.ts +++ b/server/src/metadata/workspace-migration/workspace-migration.entity.ts @@ -34,6 +34,7 @@ export type WorkspaceMigrationTableAction = { action: 'create' | 'alter'; columns?: WorkspaceMigrationColumnAction[]; }; + @Entity('workspaceMigration') export class WorkspaceMigrationEntity { @PrimaryGeneratedColumn('uuid') diff --git a/server/src/workspace/workspace-migration-runner/events/workspace-migration-applied.event.ts b/server/src/workspace/workspace-migration-runner/events/workspace-migration-applied.event.ts deleted file mode 100644 index b529b3f12..000000000 --- a/server/src/workspace/workspace-migration-runner/events/workspace-migration-applied.event.ts +++ /dev/null @@ -1,11 +0,0 @@ -export class WorkspaceMigrationAppliedEvent { - private readonly _workspaceId: string; - - constructor(worskapceId: string) { - this._workspaceId = worskapceId; - } - - get workspaceId(): string { - return this._workspaceId; - } -} diff --git a/server/src/workspace/workspace-migration-runner/events/workspace-migration-events.ts b/server/src/workspace/workspace-migration-runner/events/workspace-migration-events.ts deleted file mode 100644 index 54bb916af..000000000 --- a/server/src/workspace/workspace-migration-runner/events/workspace-migration-events.ts +++ /dev/null @@ -1,3 +0,0 @@ -export enum WorkspaceMigrationEvents { - MigrationApplied = '@workspace/migration-applied', -} diff --git a/server/src/workspace/workspace-migration-runner/workspace-migration-runner.module.ts b/server/src/workspace/workspace-migration-runner/workspace-migration-runner.module.ts index ac659ad96..96d58101b 100644 --- a/server/src/workspace/workspace-migration-runner/workspace-migration-runner.module.ts +++ b/server/src/workspace/workspace-migration-runner/workspace-migration-runner.module.ts @@ -2,11 +2,16 @@ import { Module } from '@nestjs/common'; import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module'; import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module'; +import { WorkspaceCacheVersionModule } from 'src/metadata/workspace-cache-version/workspace-cache-version.module'; import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.service'; @Module({ - imports: [WorkspaceDataSourceModule, WorkspaceMigrationModule], + imports: [ + WorkspaceDataSourceModule, + WorkspaceMigrationModule, + WorkspaceCacheVersionModule, + ], exports: [WorkspaceMigrationRunnerService], providers: [WorkspaceMigrationRunnerService], }) diff --git a/server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts b/server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts index 00f6a047f..5e43bfc6f 100644 --- a/server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts +++ b/server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts @@ -1,5 +1,4 @@ import { Injectable } from '@nestjs/common'; -import { EventEmitter2 } from '@nestjs/event-emitter'; import { QueryRunner, @@ -18,8 +17,7 @@ import { WorkspaceMigrationColumnCreate, WorkspaceMigrationColumnRelation, } from 'src/metadata/workspace-migration/workspace-migration.entity'; -import { WorkspaceMigrationEvents } from 'src/workspace/workspace-migration-runner/events/workspace-migration-events'; -import { WorkspaceMigrationAppliedEvent } from 'src/workspace/workspace-migration-runner/events/workspace-migration-applied.event'; +import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service'; import { customTableDefaultColumns } from './utils/custom-table-default-column.util'; @@ -28,7 +26,7 @@ export class WorkspaceMigrationRunnerService { constructor( private readonly workspaceDataSourceService: WorkspaceDataSourceService, private readonly workspaceMigrationService: WorkspaceMigrationService, - private readonly eventEmitter: EventEmitter2, + private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, ) {} /** @@ -82,11 +80,8 @@ export class WorkspaceMigrationRunnerService { await queryRunner.release(); - // Emit event when migration is applied - this.eventEmitter.emit( - WorkspaceMigrationEvents.MigrationApplied, - new WorkspaceMigrationAppliedEvent(workspaceId), - ); + // Increment workspace cache version + await this.workspaceCacheVersionService.incrementVersion(workspaceId); return flattenedPendingMigrations; } diff --git a/server/src/workspace/workspace-schema-storage/workspace-schema-storage.module.ts b/server/src/workspace/workspace-schema-storage/workspace-schema-storage.module.ts index 9839c8760..5728f15b0 100644 --- a/server/src/workspace/workspace-schema-storage/workspace-schema-storage.module.ts +++ b/server/src/workspace/workspace-schema-storage/workspace-schema-storage.module.ts @@ -6,11 +6,13 @@ import { MemoryStorageModule } from 'src/integrations/memory-storage/memory-stor import { MemoryStorageJsonSerializer } from 'src/integrations/memory-storage/serializers/json.serializer'; import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module'; +import { WorkspaceCacheVersionModule } from 'src/metadata/workspace-cache-version/workspace-cache-version.module'; import { WorkspaceSchemaStorageService } from 'src/workspace/workspace-schema-storage/workspace-schema-storage.service'; @Module({ imports: [ ObjectMetadataModule, + WorkspaceCacheVersionModule, MemoryStorageModule.forRoot({ identifier: 'objectMetadataCollection', type: MemoryStorageType.Local, @@ -22,6 +24,11 @@ import { WorkspaceSchemaStorageService } from 'src/workspace/workspace-schema-st type: MemoryStorageType.Local, options: {}, }), + MemoryStorageModule.forRoot({ + identifier: 'cacheVersion', + type: MemoryStorageType.Local, + options: {}, + }), ], providers: [WorkspaceSchemaStorageService], exports: [WorkspaceSchemaStorageService], diff --git a/server/src/workspace/workspace-schema-storage/workspace-schema-storage.service.ts b/server/src/workspace/workspace-schema-storage/workspace-schema-storage.service.ts index c72390293..f661d0e18 100644 --- a/server/src/workspace/workspace-schema-storage/workspace-schema-storage.service.ts +++ b/server/src/workspace/workspace-schema-storage/workspace-schema-storage.service.ts @@ -1,11 +1,9 @@ import { Injectable } from '@nestjs/common'; -import { OnEvent } from '@nestjs/event-emitter'; import { InjectMemoryStorage } from 'src/integrations/memory-storage/decorators/inject-memory-storage.decorator'; import { MemoryStorageService } from 'src/integrations/memory-storage/memory-storage.service'; import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; -import { WorkspaceMigrationAppliedEvent } from 'src/workspace/workspace-migration-runner/events/workspace-migration-applied.event'; -import { WorkspaceMigrationEvents } from 'src/workspace/workspace-migration-runner/events/workspace-migration-events'; +import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service'; @Injectable() export class WorkspaceSchemaStorageService { @@ -16,8 +14,32 @@ export class WorkspaceSchemaStorageService { >, @InjectMemoryStorage('typeDefs') private readonly typeDefsMemoryStorageService: MemoryStorageService, + @InjectMemoryStorage('cacheVersion') + private readonly cacheVersionMemoryStorageService: MemoryStorageService, + private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, ) {} + async validateCacheVersion(workspaceId: string): Promise { + const currentVersion = + (await this.cacheVersionMemoryStorageService.read({ + key: workspaceId, + })) ?? '0'; + const latestVersion = await this.workspaceCacheVersionService.getVersion( + workspaceId, + ); + + if (currentVersion !== latestVersion) { + // Invalidate cache if version mismatch is detected + await this.invalidateCache(workspaceId); + + // Update the cache version after invalidation + await this.cacheVersionMemoryStorageService.write({ + key: workspaceId, + data: latestVersion, + }); + } + } + setObjectMetadata( workspaceId: string, objectMetadata: ObjectMetadataEntity[], @@ -49,12 +71,8 @@ export class WorkspaceSchemaStorageService { }); } - /** - * Clear the workspace schema storage when new migrations are applied for a specific workspace - */ - @OnEvent(WorkspaceMigrationEvents.MigrationApplied) - handleMigrationAppliedEvent({ workspaceId }: WorkspaceMigrationAppliedEvent) { - this.objectMetadataMemoryStorageService.delete({ key: workspaceId }); - this.typeDefsMemoryStorageService.delete({ key: workspaceId }); + async invalidateCache(workspaceId: string): Promise { + await this.objectMetadataMemoryStorageService.delete({ key: workspaceId }); + await this.typeDefsMemoryStorageService.delete({ key: workspaceId }); } } diff --git a/server/src/workspace/workspace.factory.ts b/server/src/workspace/workspace.factory.ts index 638c2f06e..3b1051e99 100644 --- a/server/src/workspace/workspace.factory.ts +++ b/server/src/workspace/workspace.factory.ts @@ -39,6 +39,9 @@ export class WorkspaceFactory { return new GraphQLSchema({}); } + // Validate cache version + await this.workspaceSchemaStorageService.validateCacheVersion(workspaceId); + // Get object metadata from cache let objectMetadataCollection = await this.workspaceSchemaStorageService.getObjectMetadata(workspaceId); diff --git a/server/yarn.lock b/server/yarn.lock index 2138b935b..a7b2e5534 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -1848,13 +1848,6 @@ path-to-regexp "3.2.0" tslib "2.5.3" -"@nestjs/event-emitter@^2.0.3": - version "2.0.3" - resolved "https://registry.yarnpkg.com/@nestjs/event-emitter/-/event-emitter-2.0.3.tgz#3bfcb7b580f98b2dee3c44c567b45a506767f559" - integrity sha512-Pt7KAERrgK0OjvarSI3wfVhwZ8X1iLq1lXuodyRe+Zx3aLLP7fraFUHirASbFkB6KIQ1Zj+gZ1g8a9eu4GfFhw== - dependencies: - eventemitter2 "6.4.9" - "@nestjs/graphql@^12.0.8": version "12.0.8" resolved "https://registry.yarnpkg.com/@nestjs/graphql/-/graphql-12.0.8.tgz#15143b76dfb5fa4dc880d68a1bf2f7159ea077b6" @@ -5166,11 +5159,6 @@ etag@~1.8.1: resolved "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== -eventemitter2@6.4.9: - version "6.4.9" - resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.9.tgz#41f2750781b4230ed58827bc119d293471ecb125" - integrity sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg== - eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz"