admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: workspace cache version instead of event emitter (#2637)

Browse Source
This commit is contained in:
Jérémy M
2023-11-22 14:51:26 +01:00
committed by GitHub
parent 85646a8072
commit 5c8c141556
15 changed files with 140 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
server/package.json
View File

@ -41,7 +41,6 @@
"@nestjs/common": "^9.0.0",
"@nestjs/config": "^2.3.2",
"@nestjs/core": "^9.0.0",
"@nestjs/event-emitter": "^2.0.3",
"@nestjs/graphql": "^12.0.8",
"@nestjs/jwt": "^10.0.3",
"@nestjs/passport": "^9.0.3",

2
server/src/app.module.ts
View File

@ -2,7 +2,6 @@ import { Module } from '@nestjs/common';
import { GraphQLModule } from '@nestjs/graphql';
import { ConfigModule } from '@nestjs/config';
import { APP_FILTER, ContextIdFactory, ModuleRef } from '@nestjs/core';
import { EventEmitterModule } from '@nestjs/event-emitter';
import { YogaDriver, YogaDriverConfig } from '@graphql-yoga/nestjs';
import GraphQLJSON from 'graphql-type-json';
@ -104,7 +103,6 @@ import { ExceptionFilter } from './filters/exception.filter';
resolvers: { JSON: GraphQLJSON },
plugins: [],
}),
EventEmitterModule.forRoot(),
HealthModule,
IntegrationsModule,
CoreModule,

14
server/src/database/typeorm/metadata/migrations/1700650554672-addWorkspaceCacheVersion.ts Normal file
View File

@ -0,0 +1,14 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddWorkspaceCacheVersion1700650554672 implements MigrationInterface {
name = 'AddWorkspaceCacheVersion1700650554672'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE TABLE "metadata"."workspaceCacheVersion" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "workspaceId" character varying NOT NULL, "version" character varying NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_1a80ecf2638b477809403cc26ed" UNIQUE ("workspaceId"), CONSTRAINT "PK_5d502f8dbfb5b9a8bf2439320e9" PRIMARY KEY ("id"))`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "metadata"."workspaceCacheVersion"`);
}
}

25
server/src/metadata/workspace-cache-version/workspace-cache-version.entity.ts Normal file
View File

@ -0,0 +1,25 @@
import {
Column,
CreateDateColumn,
Entity,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm';
@Entity('workspaceCacheVersion')
export class WorkspaceCacheVersionEntity {
@PrimaryGeneratedColumn('uuid')
id: string;
@Column({ unique: true })
workspaceId: string;
@Column()
version: string;
@CreateDateColumn()
createdAt: Date;
@UpdateDateColumn()
updatedAt: Date;
}

14
server/src/metadata/workspace-cache-version/workspace-cache-version.module.ts Normal file
View File

@ -0,0 +1,14 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { WorkspaceCacheVersionEntity } from 'src/metadata/workspace-cache-version/workspace-cache-version.entity';
import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service';
@Module({
imports: [
TypeOrmModule.forFeature([WorkspaceCacheVersionEntity], 'metadata'),
],
exports: [WorkspaceCacheVersionService],
providers: [WorkspaceCacheVersionService],
})
export class WorkspaceCacheVersionModule {}

38
server/src/metadata/workspace-cache-version/workspace-cache-version.service.ts Normal file
View File

@ -0,0 +1,38 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { WorkspaceCacheVersionEntity } from 'src/metadata/workspace-cache-version/workspace-cache-version.entity';
@Injectable()
export class WorkspaceCacheVersionService {
constructor(
@InjectRepository(WorkspaceCacheVersionEntity, 'metadata')
private readonly workspaceCacheVersionRepository: Repository<WorkspaceCacheVersionEntity>,
) {}
async incrementVersion(workspaceId: string): Promise<void> {
const workspaceCacheVersion =
(await this.workspaceCacheVersionRepository.findOne({
where: { workspaceId },
})) ?? { version: '0' };
await this.workspaceCacheVersionRepository.upsert(
{
workspaceId,
version: `${+workspaceCacheVersion.version + 1}`,
},
['workspaceId'],
);
}
async getVersion(workspaceId: string): Promise<string> {
const workspaceCacheVersion =
await this.workspaceCacheVersionRepository.findOne({
where: { workspaceId },
});
return workspaceCacheVersion?.version ?? '0';
}
}

1
server/src/metadata/workspace-migration/workspace-migration.entity.ts
View File

@ -34,6 +34,7 @@ export type WorkspaceMigrationTableAction = {
action: 'create' | 'alter';
columns?: WorkspaceMigrationColumnAction[];
};
@Entity('workspaceMigration')
export class WorkspaceMigrationEntity {
@PrimaryGeneratedColumn('uuid')

11
server/src/workspace/workspace-migration-runner/events/workspace-migration-applied.event.ts
View File

@ -1,11 +0,0 @@
export class WorkspaceMigrationAppliedEvent {
private readonly _workspaceId: string;
constructor(worskapceId: string) {
this._workspaceId = worskapceId;
}
get workspaceId(): string {
return this._workspaceId;
}
}

3
server/src/workspace/workspace-migration-runner/events/workspace-migration-events.ts
View File

@ -1,3 +0,0 @@
export enum WorkspaceMigrationEvents {
MigrationApplied = '@workspace/migration-applied',
}

7
server/src/workspace/workspace-migration-runner/workspace-migration-runner.module.ts
View File

@ -2,11 +2,16 @@ import { Module } from '@nestjs/common';
import { WorkspaceMigrationModule } from 'src/metadata/workspace-migration/workspace-migration.module';
import { WorkspaceDataSourceModule } from 'src/workspace/workspace-datasource/workspace-datasource.module';
import { WorkspaceCacheVersionModule } from 'src/metadata/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.service';
@Module({
imports: [WorkspaceDataSourceModule, WorkspaceMigrationModule],
imports: [
WorkspaceDataSourceModule,
WorkspaceMigrationModule,
WorkspaceCacheVersionModule,
],
exports: [WorkspaceMigrationRunnerService],
providers: [WorkspaceMigrationRunnerService],
})

13
server/src/workspace/workspace-migration-runner/workspace-migration-runner.service.ts
View File

@ -1,5 +1,4 @@
import { Injectable } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import {
QueryRunner,
@ -18,8 +17,7 @@ import {
WorkspaceMigrationColumnCreate,
WorkspaceMigrationColumnRelation,
} from 'src/metadata/workspace-migration/workspace-migration.entity';
import { WorkspaceMigrationEvents } from 'src/workspace/workspace-migration-runner/events/workspace-migration-events';
import { WorkspaceMigrationAppliedEvent } from 'src/workspace/workspace-migration-runner/events/workspace-migration-applied.event';
import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service';
import { customTableDefaultColumns } from './utils/custom-table-default-column.util';
@ -28,7 +26,7 @@ export class WorkspaceMigrationRunnerService {
constructor(
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
private readonly workspaceMigrationService: WorkspaceMigrationService,
private readonly eventEmitter: EventEmitter2,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
) {}
/**
@ -82,11 +80,8 @@ export class WorkspaceMigrationRunnerService {
await queryRunner.release();
// Emit event when migration is applied
this.eventEmitter.emit(
WorkspaceMigrationEvents.MigrationApplied,
new WorkspaceMigrationAppliedEvent(workspaceId),
);
// Increment workspace cache version
await this.workspaceCacheVersionService.incrementVersion(workspaceId);
return flattenedPendingMigrations;
}

7
server/src/workspace/workspace-schema-storage/workspace-schema-storage.module.ts
View File

@ -6,11 +6,13 @@ import { MemoryStorageModule } from 'src/integrations/memory-storage/memory-stor
import { MemoryStorageJsonSerializer } from 'src/integrations/memory-storage/serializers/json.serializer';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { WorkspaceCacheVersionModule } from 'src/metadata/workspace-cache-version/workspace-cache-version.module';
import { WorkspaceSchemaStorageService } from 'src/workspace/workspace-schema-storage/workspace-schema-storage.service';
@Module({
imports: [
ObjectMetadataModule,
WorkspaceCacheVersionModule,
MemoryStorageModule.forRoot({
identifier: 'objectMetadataCollection',
type: MemoryStorageType.Local,
@ -22,6 +24,11 @@ import { WorkspaceSchemaStorageService } from 'src/workspace/workspace-schema-st
type: MemoryStorageType.Local,
options: {},
}),
MemoryStorageModule.forRoot({
identifier: 'cacheVersion',
type: MemoryStorageType.Local,
options: {},
}),
],
providers: [WorkspaceSchemaStorageService],
exports: [WorkspaceSchemaStorageService],

38
server/src/workspace/workspace-schema-storage/workspace-schema-storage.service.ts
View File

@ -1,11 +1,9 @@
import { Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { InjectMemoryStorage } from 'src/integrations/memory-storage/decorators/inject-memory-storage.decorator';
import { MemoryStorageService } from 'src/integrations/memory-storage/memory-storage.service';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { WorkspaceMigrationAppliedEvent } from 'src/workspace/workspace-migration-runner/events/workspace-migration-applied.event';
import { WorkspaceMigrationEvents } from 'src/workspace/workspace-migration-runner/events/workspace-migration-events';
import { WorkspaceCacheVersionService } from 'src/metadata/workspace-cache-version/workspace-cache-version.service';
@Injectable()
export class WorkspaceSchemaStorageService {
@ -16,8 +14,32 @@ export class WorkspaceSchemaStorageService {
>,
@InjectMemoryStorage('typeDefs')
private readonly typeDefsMemoryStorageService: MemoryStorageService<string>,
@InjectMemoryStorage('cacheVersion')
private readonly cacheVersionMemoryStorageService: MemoryStorageService<string>,
private readonly workspaceCacheVersionService: WorkspaceCacheVersionService,
) {}
async validateCacheVersion(workspaceId: string): Promise<void> {
const currentVersion =
(await this.cacheVersionMemoryStorageService.read({
key: workspaceId,
})) ?? '0';
const latestVersion = await this.workspaceCacheVersionService.getVersion(
workspaceId,
);
if (currentVersion !== latestVersion) {
// Invalidate cache if version mismatch is detected
await this.invalidateCache(workspaceId);
// Update the cache version after invalidation
await this.cacheVersionMemoryStorageService.write({
key: workspaceId,
data: latestVersion,
});
}
}
setObjectMetadata(
workspaceId: string,
objectMetadata: ObjectMetadataEntity[],
@ -49,12 +71,8 @@ export class WorkspaceSchemaStorageService {
});
}
/**
* Clear the workspace schema storage when new migrations are applied for a specific workspace
*/
@OnEvent(WorkspaceMigrationEvents.MigrationApplied)
handleMigrationAppliedEvent({ workspaceId }: WorkspaceMigrationAppliedEvent) {
this.objectMetadataMemoryStorageService.delete({ key: workspaceId });
this.typeDefsMemoryStorageService.delete({ key: workspaceId });
async invalidateCache(workspaceId: string): Promise<void> {
await this.objectMetadataMemoryStorageService.delete({ key: workspaceId });
await this.typeDefsMemoryStorageService.delete({ key: workspaceId });
}
}

3
server/src/workspace/workspace.factory.ts
View File

@ -39,6 +39,9 @@ export class WorkspaceFactory {
return new GraphQLSchema({});
}
// Validate cache version
await this.workspaceSchemaStorageService.validateCacheVersion(workspaceId);
// Get object metadata from cache
let objectMetadataCollection =
await this.workspaceSchemaStorageService.getObjectMetadata(workspaceId);

12
server/yarn.lock
View File

@ -1848,13 +1848,6 @@
path-to-regexp "3.2.0"
tslib "2.5.3"
"@nestjs/event-emitter@^2.0.3":
version "2.0.3"
resolved "https://registry.yarnpkg.com/@nestjs/event-emitter/-/event-emitter-2.0.3.tgz#3bfcb7b580f98b2dee3c44c567b45a506767f559"
integrity sha512-Pt7KAERrgK0OjvarSI3wfVhwZ8X1iLq1lXuodyRe+Zx3aLLP7fraFUHirASbFkB6KIQ1Zj+gZ1g8a9eu4GfFhw==
dependencies:
eventemitter2 "6.4.9"
"@nestjs/graphql@^12.0.8":
version "12.0.8"
resolved "https://registry.yarnpkg.com/@nestjs/graphql/-/graphql-12.0.8.tgz#15143b76dfb5fa4dc880d68a1bf2f7159ea077b6"
@ -5166,11 +5159,6 @@ etag@~1.8.1:
resolved "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz"
integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==
eventemitter2@6.4.9:
version "6.4.9"
resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.9.tgz#41f2750781b4230ed58827bc119d293471ecb125"
integrity sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==
eventemitter3@^3.1.0:
version "3.1.2"
resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz"