From 17a1760afd1fdc46765d191f8a851b7d76a07d3e Mon Sep 17 00:00:00 2001 From: Charles Bochet Date: Tue, 20 Aug 2024 19:42:02 +0200 Subject: [PATCH] Improve performance twenty orm (#6691) ## Context As we grow, the messaging scripts are experiencing performance issues forcing us to temporarily disable them on the cloud. While investigating the performance, I have noticed that generating the entity schema (for twentyORM) in the repository is taking ~500ms locally on my Mac M2 so likely more on pods. Caching the entitySchema then! I'm also clarifying naming around schemaVersion and cacheVersions ==> both are renamed workspaceMetadataVersion and migrated to the workspace table (the workspaceCacheVersion table is dropped). --- .../src/generated/graphql.tsx | 2 +- .../src/generated-metadata/graphql.ts | 5 +- .../twenty-front/src/generated/graphql.tsx | 14 +- .../modules/apollo/hooks/useApolloFactory.ts | 8 +- .../auth/states/currentWorkspaceState.ts | 2 +- .../graphql/fragments/userQueryFragment.ts | 2 +- .../src/testing/mock-data/users.ts | 4 +- packages/twenty-server/@types/express.d.ts | 2 +- ...hql-yoga-nestjs-npm-2.1.0-cb509e6047.patch | 12 +- packages/twenty-server/src/app.module.ts | 20 +-- .../data-seed-dev-workspace.command.ts | 10 +- .../commands/database-command.module.ts | 4 +- ...essage-channel-sync-status-enum.command.ts | 8 +- ...user-vars-accounts-to-reconnect.command.ts | 8 +- ...set-workspace-activation-status.command.ts | 8 +- .../0-23/0-23-update-activities.command.ts | 8 +- .../0-23/0-23-upgrade-version.module.ts | 4 +- .../upgrade-version/upgrade-version.module.ts | 4 +- ...MetadataVersionAndDatasourceOnWorkspace.ts | 31 +++++ .../1724173061204-deprecateCacheVersion.ts | 15 +++ .../__tests__/workspace.factory.spec.ts | 9 +- .../api/graphql/core-graphql-api.module.ts | 6 +- .../hooks/use-cached-metadata.ts | 5 +- .../graphql/metadata-graphql-api.module.ts | 16 +-- .../api/graphql/metadata.module-factory.ts | 10 +- .../api/graphql/workspace-schema.factory.ts | 22 +-- .../google-apis-auth.controller.ts | 18 +-- .../auth/services/google-apis.service.ts | 16 ++- .../auth/strategies/google.auth.strategy.ts | 6 +- .../strategies/microsoft.auth.strategy.ts | 2 +- .../auth/types/google-api-request.type.ts | 2 +- .../file/controllers/file.controller.ts | 4 + .../hooks/use-graphql-error-handler.hook.ts | 8 +- .../engine/core-modules/user/user.resolver.ts | 9 +- .../workspace/workspace.entity.ts | 12 ++ .../workspace/workspace.module.ts | 4 +- .../workspace/workspace.resolver.ts | 23 ++-- .../types/cache-storage-namespace.enum.ts | 6 +- .../field-metadata/field-metadata.module.ts | 6 +- .../field-metadata/field-metadata.service.ts | 16 ++- .../metadata-engine.module.ts | 6 +- .../object-metadata/object-metadata.module.ts | 4 +- .../object-metadata.service.ts | 18 ++- .../relation-metadata.module.ts | 4 +- .../relation-metadata.service.ts | 10 +- .../foreign-table/foreign-table.module.ts | 4 +- .../foreign-table/foreign-table.service.ts | 8 +- .../remote-table/remote-table.module.ts | 4 +- .../remote-table/remote-table.service.ts | 66 ++++----- .../workspace-cache-version.entity.ts | 25 ---- .../workspace-cache-version.module.ts | 14 -- .../workspace-cache-version.service.ts | 42 ------ .../workspace-metadata-version.module.ts | 16 +++ .../workspace-metadata-version.service.ts | 79 +++++++++++ ...l-hydrate-request-from-token.middleware.ts | 13 +- .../load-service-with-workspace.context.ts | 44 ------ .../scoped-workspace-context.factory.ts | 8 +- .../factories/workspace-datasource.factory.ts | 77 ++++++++--- .../twenty-orm/twenty-orm-core.module.ts | 13 +- .../engine/twenty-orm/twenty-orm.manager.ts | 11 +- .../workspace-cache-storage.module.ts | 2 - .../workspace-cache-storage.service.ts | 125 +++++++++++------- .../commands/delete-workspaces.command.ts | 22 ++- .../workspace-migration-runner.module.ts | 7 +- .../workspace-sync-metadata.module.ts | 4 +- .../workspace-sync-metadata.service.ts | 6 +- .../calendar-event-list-fetch.cron.job.ts | 4 + .../jobs/calendar-event-list-fetch.job.ts | 3 + .../calendar-channel-sync-status.service.ts | 2 +- .../messaging-channel-sync-status.service.ts | 2 +- .../messaging-message-list-fetch.cron.job.ts | 4 + .../messaging-messages-import.cron.job.ts | 3 + ...-single-message-to-cache-for-import.job.ts | 2 +- .../jobs/messaging-clean-cache.ts | 10 +- .../jobs/messaging-message-list-fetch.job.ts | 4 + .../jobs/messaging-messages-import.job.ts | 4 + ...ssaging-full-message-list-fetch.service.ts | 2 +- .../messaging-messages-import.service.ts | 2 +- ...ging-partial-message-list-fetch.service.ts | 2 +- ...age-channel-sync-status-monitoring.cron.ts | 4 + 80 files changed, 583 insertions(+), 468 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts create mode 100644 packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts delete mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts create mode 100644 packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts delete mode 100644 packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts diff --git a/packages/twenty-chrome-extension/src/generated/graphql.tsx b/packages/twenty-chrome-extension/src/generated/graphql.tsx index b63bc43e6..27d954014 100644 --- a/packages/twenty-chrome-extension/src/generated/graphql.tsx +++ b/packages/twenty-chrome-extension/src/generated/graphql.tsx @@ -7455,7 +7455,7 @@ export type Workspace = { billingSubscriptions?: Maybe>; createdAt: Scalars['DateTime']; currentBillingSubscription?: Maybe; - currentCacheVersion?: Maybe; + metadataVersion?: Maybe; deletedAt?: Maybe; displayName?: Maybe; domainName?: Maybe; diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index f5e61e07a..ca9941fc6 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -1380,7 +1380,9 @@ export type Workspace = { billingSubscriptions?: Maybe>; createdAt: Scalars['DateTime']['output']; currentBillingSubscription?: Maybe; - currentCacheVersion?: Maybe; + currentMetadataVersion: Scalars['Float']['output']; + databaseSchema: Scalars['String']['output']; + databaseUrl: Scalars['String']['output']; deletedAt?: Maybe; displayName?: Maybe; domainName?: Maybe; @@ -1388,6 +1390,7 @@ export type Workspace = { id: Scalars['UUID']['output']; inviteHash?: Maybe; logo?: Maybe; + metadataVersion: Scalars['Float']['output']; updatedAt: Scalars['DateTime']['output']; workspaceMembersCount?: Maybe; }; diff --git a/packages/twenty-front/src/generated/graphql.tsx b/packages/twenty-front/src/generated/graphql.tsx index cb973a45c..45bff9ca1 100644 --- a/packages/twenty-front/src/generated/graphql.tsx +++ b/packages/twenty-front/src/generated/graphql.tsx @@ -1067,7 +1067,8 @@ export type Workspace = { billingSubscriptions?: Maybe>; createdAt: Scalars['DateTime']; currentBillingSubscription?: Maybe; - currentCacheVersion?: Maybe; + databaseSchema: Scalars['String']; + databaseUrl: Scalars['String']; deletedAt?: Maybe; displayName?: Maybe; domainName?: Maybe; @@ -1075,6 +1076,7 @@ export type Workspace = { id: Scalars['UUID']; inviteHash?: Maybe; logo?: Maybe; + metadataVersion: Scalars['Float']; updatedAt: Scalars['DateTime']; workspaceMembersCount?: Maybe; }; @@ -1360,7 +1362,7 @@ export type ImpersonateMutationVariables = Exact<{ }>; -export type ImpersonateMutation = { __typename?: 'Mutation', impersonate: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; +export type ImpersonateMutation = { __typename?: 'Mutation', impersonate: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; export type RenewTokenMutationVariables = Exact<{ appToken: Scalars['String']; @@ -1392,7 +1394,7 @@ export type VerifyMutationVariables = Exact<{ }>; -export type VerifyMutation = { __typename?: 'Mutation', verify: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; +export type VerifyMutation = { __typename?: 'Mutation', verify: { __typename?: 'Verify', user: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }, tokens: { __typename?: 'AuthTokenPair', accessToken: { __typename?: 'AuthToken', token: string, expiresAt: string }, refreshToken: { __typename?: 'AuthToken', token: string, expiresAt: string } } } }; export type CheckUserExistsQueryVariables = Exact<{ email: Scalars['String']; @@ -1453,7 +1455,7 @@ export type GetAisqlQueryQueryVariables = Exact<{ export type GetAisqlQueryQuery = { __typename?: 'Query', getAISQLQuery: { __typename?: 'AISQLQueryResult', sqlQuery: string, sqlQueryResult?: string | null, queryFailedErrorMessage?: string | null } }; -export type UserQueryFragmentFragment = { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }; +export type UserQueryFragmentFragment = { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> }; export type DeleteUserAccountMutationVariables = Exact<{ [key: string]: never; }>; @@ -1470,7 +1472,7 @@ export type UploadProfilePictureMutation = { __typename?: 'Mutation', uploadProf export type GetCurrentUserQueryVariables = Exact<{ [key: string]: never; }>; -export type GetCurrentUserQuery = { __typename?: 'Query', currentUser: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, currentCacheVersion?: string | null, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> } }; +export type GetCurrentUserQuery = { __typename?: 'Query', currentUser: { __typename?: 'User', id: any, firstName: string, lastName: string, email: string, canImpersonate: boolean, supportUserHash?: string | null, onboardingStatus?: OnboardingStatus | null, userVars: any, workspaceMember?: { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } } | null, workspaceMembers?: Array<{ __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }> | null, defaultWorkspace: { __typename?: 'Workspace', id: any, displayName?: string | null, logo?: string | null, domainName?: string | null, inviteHash?: string | null, allowImpersonation: boolean, activationStatus: WorkspaceActivationStatus, metadataVersion: number, workspaceMembersCount?: number | null, featureFlags?: Array<{ __typename?: 'FeatureFlag', id: any, key: string, value: boolean, workspaceId: string }> | null, currentBillingSubscription?: { __typename?: 'BillingSubscription', id: any, status: SubscriptionStatus, interval?: SubscriptionInterval | null } | null }, workspaces: Array<{ __typename?: 'UserWorkspace', workspace?: { __typename?: 'Workspace', id: any, logo?: string | null, displayName?: string | null, domainName?: string | null } | null }> } }; export type WorkspaceMemberQueryFragmentFragment = { __typename?: 'WorkspaceMember', id: any, colorScheme: string, avatarUrl?: string | null, locale?: string | null, timeZone?: string | null, dateFormat?: WorkspaceMemberDateFormatEnum | null, timeFormat?: WorkspaceMemberTimeFormatEnum | null, name: { __typename?: 'FullName', firstName: string, lastName: string } }; @@ -1652,7 +1654,7 @@ export const UserQueryFragmentFragmentDoc = gql` value workspaceId } - currentCacheVersion + metadataVersion currentBillingSubscription { id status diff --git a/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts b/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts index 351ac8289..b70cbdd17 100644 --- a/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts +++ b/packages/twenty-front/src/modules/apollo/hooks/useApolloFactory.ts @@ -1,6 +1,6 @@ +import { InMemoryCache, NormalizedCacheObject } from '@apollo/client'; import { useMemo, useRef } from 'react'; import { useLocation, useNavigate } from 'react-router-dom'; -import { InMemoryCache, NormalizedCacheObject } from '@apollo/client'; import { useRecoilState, useSetRecoilState } from 'recoil'; import { currentUserState } from '@/auth/states/currentUserState'; @@ -49,8 +49,8 @@ export const useApolloFactory = (options: Partial> = {}) => { }, }), headers: { - ...(currentWorkspace?.currentCacheVersion && { - 'X-Schema-Version': currentWorkspace.currentCacheVersion, + ...(currentWorkspace?.metadataVersion && { + 'X-Schema-Version': `${currentWorkspace.metadataVersion}`, }), }, defaultOptions: { @@ -95,7 +95,7 @@ export const useApolloFactory = (options: Partial> = {}) => { setCurrentWorkspace, setWorkspaces, isDebugMode, - currentWorkspace?.currentCacheVersion, + currentWorkspace?.metadataVersion, setPreviousUrl, ]); diff --git a/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts b/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts index a66d7bd1c..8dd9ea7bc 100644 --- a/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts +++ b/packages/twenty-front/src/modules/auth/states/currentWorkspaceState.ts @@ -13,7 +13,7 @@ export type CurrentWorkspace = Pick< | 'activationStatus' | 'currentBillingSubscription' | 'workspaceMembersCount' - | 'currentCacheVersion' + | 'metadataVersion' >; export const currentWorkspaceState = createState({ diff --git a/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts b/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts index 0d1971e29..76e5400c3 100644 --- a/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts +++ b/packages/twenty-front/src/modules/users/graphql/fragments/userQueryFragment.ts @@ -29,7 +29,7 @@ export const USER_QUERY_FRAGMENT = gql` value workspaceId } - currentCacheVersion + metadataVersion currentBillingSubscription { id status diff --git a/packages/twenty-front/src/testing/mock-data/users.ts b/packages/twenty-front/src/testing/mock-data/users.ts index 5d6514c42..f7a4c2727 100644 --- a/packages/twenty-front/src/testing/mock-data/users.ts +++ b/packages/twenty-front/src/testing/mock-data/users.ts @@ -63,7 +63,7 @@ export const mockDefaultWorkspace: Workspace = { ], createdAt: '2023-04-26T10:23:42.33625+00:00', updatedAt: '2023-04-26T10:23:42.33625+00:00', - currentCacheVersion: '1', + metadataVersion: 1, currentBillingSubscription: { __typename: 'BillingSubscription', id: '7efbc3f7-6e5e-4128-957e-8d86808cdf6a', @@ -71,6 +71,8 @@ export const mockDefaultWorkspace: Workspace = { status: SubscriptionStatus.Active, }, workspaceMembersCount: 1, + databaseSchema: '', + databaseUrl: '', }; export const mockedWorkspaceMemberData: WorkspaceMember = { diff --git a/packages/twenty-server/@types/express.d.ts b/packages/twenty-server/@types/express.d.ts index 89fea7d77..1e45cf88d 100644 --- a/packages/twenty-server/@types/express.d.ts +++ b/packages/twenty-server/@types/express.d.ts @@ -8,6 +8,6 @@ declare module 'express-serve-static-core' { apiKey?: ApiKeyWorkspaceEntity | null; workspace?: Workspace; workspaceId?: string; - cacheVersion?: string | null; + workspaceMetadataVersion?: number; } } diff --git a/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch b/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch index 70572a490..b15363c16 100644 --- a/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch +++ b/packages/twenty-server/patches/@graphql-yoga-nestjs-npm-2.1.0-cb509e6047.patch @@ -32,7 +32,7 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -79,7 +79,7 @@ index 16843949d8589a299d8195b0a349ac4dac0bacbf..21e7fe2bbcba36b04a274be9d2219fd3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -146,7 +146,7 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -193,7 +193,7 @@ index 7068c519320b379917c46763cd280b1cdd3e48f0..418e1030373fc1e0fb85a932ac8da9b3 ...options, + schema: async (request) => { + const workspaceId = request.req.workspace?.id ?? 'anonymous' -+ const workspaceCacheVersion = request.req.cacheVersion ?? '0' ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion ?? '0' + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -387,7 +387,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac ...options, + schema: async request => { + const workspaceId = request.req.workspace.id -+ const workspaceCacheVersion = request.req.cacheVersion ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + @@ -448,7 +448,7 @@ index ce142f61ede52499485b19d8af057f4cb828d0f7..5888d31cae1b7aca57ed0819209812ac ...options, + schema: async request => { + const workspaceId = request.req.workspace.id -+ const workspaceCacheVersion = request.req.cacheVersion ++ const workspaceCacheVersion = request.req.workspaceMetadataVersion + const workspaceUserId = request.req.user?.id ?? 'anonymous' + const url = request.req.baseUrl + diff --git a/packages/twenty-server/src/app.module.ts b/packages/twenty-server/src/app.module.ts index 914b5b295..6e7b5316e 100644 --- a/packages/twenty-server/src/app.module.ts +++ b/packages/twenty-server/src/app.module.ts @@ -5,27 +5,27 @@ import { RequestMethod, } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; -import { ServeStaticModule } from '@nestjs/serve-static'; import { GraphQLModule } from '@nestjs/graphql'; +import { ServeStaticModule } from '@nestjs/serve-static'; import { existsSync } from 'fs'; import { join } from 'path'; -import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs'; +import { YogaDriver, YogaDriverConfig } from '@graphql-yoga/nestjs'; -import { RestApiModule } from 'src/engine/api/rest/rest-api.module'; -import { ModulesModule } from 'src/modules/modules.module'; import { CoreGraphQLApiModule } from 'src/engine/api/graphql/core-graphql-api.module'; -import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphql-api.module'; import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module'; import { GraphQLConfigService } from 'src/engine/api/graphql/graphql-config/graphql-config.service'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; -import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware'; -import { MessageQueueModule } from 'src/engine/integrations/message-queue/message-queue.module'; +import { MetadataGraphQLApiModule } from 'src/engine/api/graphql/metadata-graphql-api.module'; +import { RestApiModule } from 'src/engine/api/rest/rest-api.module'; import { MessageQueueDriverType } from 'src/engine/integrations/message-queue/interfaces'; +import { MessageQueueModule } from 'src/engine/integrations/message-queue/message-queue.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware'; +import { ModulesModule } from 'src/modules/modules.module'; -import { IntegrationsModule } from './engine/integrations/integrations.module'; import { CoreEngineModule } from './engine/core-modules/core-engine.module'; +import { IntegrationsModule } from './engine/integrations/integrations.module'; @Module({ imports: [ @@ -52,7 +52,7 @@ import { CoreEngineModule } from './engine/core-modules/core-engine.module'; // Modules module, contains all business logic modules ModulesModule, // Needed for the user workspace middleware - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, // Api modules CoreGraphQLApiModule, MetadataGraphQLApiModule, diff --git a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts index 2152c7878..6e5fef348 100644 --- a/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts +++ b/packages/twenty-server/src/database/commands/data-seed-dev-workspace.command.ts @@ -1,8 +1,7 @@ import { Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; import { Command, CommandRunner } from 'nest-commander'; -import { EntityManager, Repository } from 'typeorm'; +import { EntityManager } from 'typeorm'; import { seedCoreSchema } from 'src/database/typeorm-seeds/core'; import { @@ -39,7 +38,6 @@ import { DataSourceService } from 'src/engine/metadata-modules/data-source/data- import { FieldMetadataService } from 'src/engine/metadata-modules/field-metadata/field-metadata.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service'; import { viewPrefillData } from 'src/engine/workspace-manager/standard-objects-prefill-data/view'; import { STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids'; @@ -61,12 +59,9 @@ export class DataSeedWorkspaceCommand extends CommandRunner { private readonly workspaceSyncMetadataService: WorkspaceSyncMetadataService, private readonly workspaceDataSourceService: WorkspaceDataSourceService, private readonly fieldMetadataService: FieldMetadataService, - @InjectRepository(ObjectMetadataEntity, 'metadata') - private readonly objectMetadataRepository: Repository, private readonly objectMetadataService: ObjectMetadataService, - @InjectCacheStorage(CacheStorageNamespace.WorkspaceSchema) + @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) private readonly workspaceSchemaCache: CacheStorageService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, ) { super(); } @@ -75,7 +70,6 @@ export class DataSeedWorkspaceCommand extends CommandRunner { try { for (const workspaceId of this.workspaceIds) { await this.workspaceSchemaCache.flush(); - await this.workspaceCacheVersionService.deleteVersion(workspaceId); await rawDataSource.initialize(); diff --git a/packages/twenty-server/src/database/commands/database-command.module.ts b/packages/twenty-server/src/database/commands/database-command.module.ts index 80d685c95..6bf31eb73 100644 --- a/packages/twenty-server/src/database/commands/database-command.module.ts +++ b/packages/twenty-server/src/database/commands/database-command.module.ts @@ -19,7 +19,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; @@ -45,7 +45,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp ObjectMetadataModule, FieldMetadataModule, DataSeedDemoWorkspaceModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, UpgradeTo0_23CommandModule, UpgradeVersionModule, ], diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts index 47e45fc0a..5e31d61c2 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-migrate-message-channel-sync-status-enum.command.ts @@ -10,7 +10,7 @@ import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceStatusService } from 'src/engine/workspace-manager/workspace-status/services/workspace-status.service'; import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @@ -34,7 +34,7 @@ export class MigrateMessageChannelSyncStatusEnumCommand extends CommandRunner { private readonly objectMetadataRepository: Repository, private readonly typeORMService: TypeORMService, private readonly dataSourceService: DataSourceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, ) { super(); } @@ -212,7 +212,9 @@ export class MigrateMessageChannelSyncStatusEnumCommand extends CommandRunner { options: newOptions, }); - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); this.logger.log( chalk.green(`Running command on workspace ${workspaceId} done`), diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts index 875a9fdaa..726d698ea 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-user-vars-accounts-to-reconnect.command.ts @@ -13,7 +13,7 @@ import { Workspace, WorkspaceActivationStatus, } from 'src/engine/core-modules/workspace/workspace.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity'; import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service'; @@ -34,7 +34,7 @@ export class SetUserVarsAccountsToReconnectCommand extends CommandRunner { SetUserVarsAccountsToReconnectCommand.name, ); constructor( - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, private readonly accountsToReconnectService: AccountsToReconnectService, @InjectRepository(KeyValuePair, 'core') @@ -149,7 +149,9 @@ export class SetUserVarsAccountsToReconnectCommand extends CommandRunner { throw error; } - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); this.logger.log( chalk.green(`Running command on workspace ${workspaceId} done`), diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts index 7331b7fde..8021a4183 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-set-workspace-activation-status.command.ts @@ -12,7 +12,7 @@ import { WorkspaceActivationStatus, } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; interface SetWorkspaceActivationStatusCommandOptions { workspaceId?: string; @@ -31,7 +31,7 @@ export class SetWorkspaceActivationStatusCommand extends CommandRunner { private readonly workspaceRepository: Repository, private readonly typeORMService: TypeORMService, private readonly dataSourceService: DataSourceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly billingSubscriptionService: BillingSubscriptionService, ) { super(); @@ -97,7 +97,9 @@ export class SetWorkspaceActivationStatusCommand extends CommandRunner { } } - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); this.logger.log( chalk.green(`Running command on workspace ${workspaceId} done`), diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts index fcc8b7b3c..da4db435a 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-update-activities.command.ts @@ -9,7 +9,7 @@ import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { notesAllView } from 'src/engine/workspace-manager/standard-objects-prefill-data/views/notes-all.view'; import { tasksAllView } from 'src/engine/workspace-manager/standard-objects-prefill-data/views/tasks-all.view'; @@ -44,7 +44,7 @@ export class UpdateActivitiesCommand extends CommandRunner { private readonly workspaceStatusService: WorkspaceStatusService, private readonly typeORMService: TypeORMService, private readonly dataSourceService: DataSourceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly objectMetadataService: ObjectMetadataService, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, ) { @@ -400,7 +400,9 @@ export class UpdateActivitiesCommand extends CommandRunner { .execute(); } - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); } } diff --git a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts index e62f15b65..09fbe2fb9 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/0-23/0-23-upgrade-version.module.ts @@ -21,7 +21,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module'; import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module'; @@ -35,7 +35,7 @@ import { ViewModule } from 'src/modules/view/view.module'; OnboardingModule, TypeORMModule, DataSourceModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, FieldMetadataModule, DataSourceModule, WorkspaceStatusModule, diff --git a/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts b/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts index d0d869b40..2cfefd9d0 100644 --- a/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts +++ b/packages/twenty-server/src/database/commands/upgrade-version/upgrade-version.module.ts @@ -21,7 +21,7 @@ import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/ import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; @@ -48,7 +48,7 @@ import { ViewModule } from 'src/modules/view/view.module'; WorkspaceStatusModule, ObjectMetadataModule, DataSeedDemoWorkspaceModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, FieldMetadataModule, ViewModule, BillingModule, diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts b/packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts new file mode 100644 index 000000000..f53ccd85f --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/migrations/1724173430043-introduceMetadataVersionAndDatasourceOnWorkspace.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class IntroduceMetadataVersionAndDatasourceOnWorkspace1724173430043 + implements MigrationInterface +{ + name = 'IntroduceMetadataVersionAndDatasourceOnWorkspace1724173430043'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."workspace" ADD "metadataVersion" integer NOT NULL DEFAULT '1'`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" ADD "databaseUrl" character varying NOT NULL DEFAULT ''`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" ADD "databaseSchema" character varying NOT NULL DEFAULT ''`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."workspace" DROP COLUMN "databaseSchema"`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" DROP COLUMN "databaseUrl"`, + ); + await queryRunner.query( + `ALTER TABLE "core"."workspace" DROP COLUMN "metadataVersion"`, + ); + } +} diff --git a/packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts b/packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts new file mode 100644 index 000000000..37a5d6674 --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/metadata/migrations/1724173061204-deprecateCacheVersion.ts @@ -0,0 +1,15 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class DeprecateCacheVersion1724173061204 implements MigrationInterface { + name = 'DeprecateCacheVersion1724173061204'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "metadata"."workspaceCacheVersion"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "metadata"."workspaceCacheVersion" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "workspaceId" uuid NOT NULL, "version" character varying NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_1a80ecf2638b477809403cc26ed" UNIQUE ("workspaceId"), CONSTRAINT "PK_5d502f8dbfb5b9a8bf2439320e9" PRIMARY KEY ("id"))`, + ); + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts index f3554cdde..57d051363 100644 --- a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts +++ b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts @@ -1,11 +1,12 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory'; import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; import { WorkspaceSchemaFactory } from 'src/engine/api/graphql/workspace-schema.factory'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; describe('WorkspaceSchemaFactory', () => { @@ -39,6 +40,10 @@ describe('WorkspaceSchemaFactory', () => { provide: WorkspaceCacheStorageService, useValue: {}, }, + { + provide: WorkspaceMetadataVersionService, + useValue: {}, + }, ], }).compile(); diff --git a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts index 33c24c621..c9b0aa852 100644 --- a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts @@ -1,10 +1,11 @@ import { Module } from '@nestjs/common'; -import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; -import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module'; import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver-builder.module'; +import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module'; import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; +import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceSchemaFactory } from './workspace-schema.factory'; @@ -21,6 +22,7 @@ import { WorkspaceSchemaFactory } from './workspace-schema.factory'; WorkspaceSchemaBuilderModule, WorkspaceResolverBuilderModule, WorkspaceCacheStorageModule, + WorkspaceMetadataVersionModule, ], providers: [WorkspaceSchemaFactory, ScalarsExplorerService], exports: [WorkspaceSchemaFactory], diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts index 3e3be6b1b..d8b7de5d6 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-config/hooks/use-cached-metadata.ts @@ -9,10 +9,11 @@ export type CacheMetadataPluginConfig = { export function useCachedMetadata(config: CacheMetadataPluginConfig): Plugin { const computeCacheKey = (serverContext: any) => { const workspaceId = serverContext.req.workspace?.id ?? 'anonymous'; - const cacheVersion = serverContext.req.cacheVersion ?? '0'; + const workspaceMetadataVersion = + serverContext.req.workspaceMetadataVersion ?? '0'; const operationName = getOperationName(serverContext); - return `cachedOperations:${operationName}:${workspaceId}:${cacheVersion}`; + return `graphql:operations:${operationName}:${workspaceId}:${workspaceMetadataVersion}`; }; const getOperationName = (serverContext: any) => diff --git a/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts b/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts index cc41b6e74..3d0627ce7 100644 --- a/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/metadata-graphql-api.module.ts @@ -1,19 +1,18 @@ import { Module } from '@nestjs/common'; import { GraphQLModule } from '@nestjs/graphql'; -import { YogaDriverConfig, YogaDriver } from '@graphql-yoga/nestjs'; +import { YogaDriver, YogaDriverConfig } from '@graphql-yoga/nestjs'; -import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; -import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; -import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; import { GraphQLConfigModule } from 'src/engine/api/graphql/graphql-config/graphql-config.module'; import { metadataModuleFactory } from 'src/engine/api/graphql/metadata.module-factory'; -import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service'; import { DataloaderModule } from 'src/engine/dataloaders/dataloader.module'; import { DataloaderService } from 'src/engine/dataloaders/dataloader.service'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; -import { CacheStorageModule } from 'src/engine/integrations/cache-storage/cache-storage.module'; +import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; +import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service'; +import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; +import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; +import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @Module({ imports: [ @@ -25,13 +24,12 @@ import { CacheStorageModule } from 'src/engine/integrations/cache-storage/cache- EnvironmentService, ExceptionHandlerService, DataloaderService, - CacheStorageNamespace.WorkspaceSchema, + CacheStorageNamespace.EngineWorkspace, ], }), MetadataEngineModule, WorkspaceMigrationRunnerModule, WorkspaceMigrationModule, - CacheStorageModule, ], }) export class MetadataGraphQLApiModule {} diff --git a/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts b/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts index 3d2e2b7e1..5297b747b 100644 --- a/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/metadata.module-factory.ts @@ -15,7 +15,7 @@ export const metadataModuleFactory = async ( environmentService: EnvironmentService, exceptionHandlerService: ExceptionHandlerService, dataloaderService: DataloaderService, - workspaceSchemaCacheStorage: CacheStorageService, + cacheStorageService: CacheStorageService, ): Promise => { const config: YogaDriverConfig = { autoSchemaFile: true, @@ -36,12 +36,8 @@ export const metadataModuleFactory = async ( exceptionHandlerService, }), useCachedMetadata({ - cacheGetter: workspaceSchemaCacheStorage.get.bind( - workspaceSchemaCacheStorage, - ), - cacheSetter: workspaceSchemaCacheStorage.set.bind( - workspaceSchemaCacheStorage, - ), + cacheGetter: cacheStorageService.get.bind(cacheStorageService), + cacheSetter: cacheStorageService.set.bind(cacheStorageService), operationsToCache: ['ObjectMetadataItems'], }), ], diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts index f06c94c20..c82f3d18f 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts @@ -1,17 +1,18 @@ import { Injectable } from '@nestjs/common'; -import { GraphQLSchema, printSchema } from 'graphql'; import { makeExecutableSchema } from '@graphql-tools/schema'; +import { GraphQLSchema, printSchema } from 'graphql'; import { gql } from 'graphql-tag'; -import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; -import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; import { ScalarsExplorerService } from 'src/engine/api/graphql/services/scalars-explorer.service'; -import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; import { workspaceResolverBuilderMethodNames } from 'src/engine/api/graphql/workspace-resolver-builder/factories/factories'; import { WorkspaceResolverFactory } from 'src/engine/api/graphql/workspace-resolver-builder/workspace-resolver.factory'; +import { WorkspaceGraphQLSchemaFactory } from 'src/engine/api/graphql/workspace-schema-builder/workspace-graphql-schema.factory'; import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; +import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @Injectable() export class WorkspaceSchemaFactory { @@ -22,6 +23,7 @@ export class WorkspaceSchemaFactory { private readonly workspaceGraphQLSchemaFactory: WorkspaceGraphQLSchemaFactory, private readonly workspaceResolverFactory: WorkspaceResolverFactory, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, ) {} async createGraphQLSchema(authContext: AuthContext): Promise { @@ -40,7 +42,7 @@ export class WorkspaceSchemaFactory { } // Validate cache version - await this.workspaceCacheStorageService.validateCacheVersion( + await this.workspaceMetadataVersionService.flushCacheIfMetadataVersionIsOutdated( authContext.workspace.id, ); @@ -64,11 +66,11 @@ export class WorkspaceSchemaFactory { } // Get typeDefs from cache - let typeDefs = await this.workspaceCacheStorageService.getTypeDefs( + let typeDefs = await this.workspaceCacheStorageService.getGraphQLTypeDefs( authContext.workspace.id, ); let usedScalarNames = - await this.workspaceCacheStorageService.getUsedScalarNames( + await this.workspaceCacheStorageService.getGraphQLUsedScalarNames( authContext.workspace.id, ); @@ -84,11 +86,11 @@ export class WorkspaceSchemaFactory { this.scalarsExplorerService.getUsedScalarNames(autoGeneratedSchema); typeDefs = printSchema(autoGeneratedSchema); - await this.workspaceCacheStorageService.setTypeDefs( + await this.workspaceCacheStorageService.setGraphQLTypeDefs( authContext.workspace.id, typeDefs, ); - await this.workspaceCacheStorageService.setUsedScalarNames( + await this.workspaceCacheStorageService.setGraphQLUsedScalarNames( authContext.workspace.id, usedScalarNames, ); diff --git a/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts b/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts index de2fe4750..a2fe6db48 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/controllers/google-apis-auth.controller.ts @@ -21,7 +21,6 @@ import { TokenService } from 'src/engine/core-modules/auth/services/token.servic import { GoogleAPIsRequest } from 'src/engine/core-modules/auth/types/google-api-request.type'; import { OnboardingService } from 'src/engine/core-modules/onboarding/onboarding.service'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; @Controller('auth/google-apis') @UseFilters(AuthRestApiExceptionFilter) @@ -31,7 +30,6 @@ export class GoogleAPIsAuthController { private readonly tokenService: TokenService, private readonly environmentService: EnvironmentService, private readonly onboardingService: OnboardingService, - private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext, ) {} @Get() @@ -80,13 +78,7 @@ export class GoogleAPIsAuthController { const handle = emails[0].value; - const googleAPIsServiceInstance = - await this.loadServiceWithWorkspaceContext.load( - this.googleAPIsService, - workspaceId, - ); - - await googleAPIsServiceInstance.refreshGoogleRefreshToken({ + await this.googleAPIsService.refreshGoogleRefreshToken({ handle, workspaceMemberId: workspaceMemberId, workspaceId: workspaceId, @@ -97,13 +89,7 @@ export class GoogleAPIsAuthController { }); if (userId) { - const onboardingServiceInstance = - await this.loadServiceWithWorkspaceContext.load( - this.onboardingService, - workspaceId, - ); - - await onboardingServiceInstance.setOnboardingConnectAccountPending({ + await this.onboardingService.setOnboardingConnectAccountPending({ userId, workspaceId, value: false, diff --git a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts index 157c82ab7..dcee921d2 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/services/google-apis.service.ts @@ -8,7 +8,7 @@ import { InjectMessageQueue } from 'src/engine/integrations/message-queue/decora import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; -import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; +import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { CalendarEventListFetchJob, CalendarEventsImportJobData, @@ -39,7 +39,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta @Injectable() export class GoogleAPIsService { constructor( - private readonly twentyORMManager: TwentyORMManager, + private readonly twentyORMGlobalManager: TwentyORMGlobalManager, @InjectMessageQueue(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, @InjectMessageQueue(MessageQueue.calendarQueue) @@ -82,16 +82,19 @@ export class GoogleAPIsService { const newOrExistingConnectedAccountId = existingAccountId ?? v4(); const calendarChannelRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'calendarChannel', ); const messageChannelRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'messageChannel', ); - const workspaceDataSource = await this.twentyORMManager.getDatasource(); + const workspaceDataSource = + await this.twentyORMGlobalManager.getDataSourceForWorkspace(workspaceId); await workspaceDataSource.transaction(async (manager: EntityManager) => { if (!existingAccountId) { @@ -146,7 +149,8 @@ export class GoogleAPIsService { ); const workspaceMemberRepository = - await this.twentyORMManager.getRepository( + await this.twentyORMGlobalManager.getRepositoryForWorkspace( + workspaceId, 'workspaceMember', ); diff --git a/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts b/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts index ec5669808..6cf51366f 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/strategies/google.auth.strategy.ts @@ -1,14 +1,14 @@ -import { PassportStrategy } from '@nestjs/passport'; import { Injectable } from '@nestjs/common'; +import { PassportStrategy } from '@nestjs/passport'; -import { Strategy, VerifyCallback } from 'passport-google-oauth20'; import { Request } from 'express'; +import { Strategy, VerifyCallback } from 'passport-google-oauth20'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; export type GoogleRequest = Omit< Request, - 'user' | 'workspace' | 'cacheVersion' + 'user' | 'workspace' | 'workspaceMetadataVersion' > & { user: { firstName?: string | null; diff --git a/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts b/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts index f2d002a78..8eae65a81 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/strategies/microsoft.auth.strategy.ts @@ -12,7 +12,7 @@ import { EnvironmentService } from 'src/engine/integrations/environment/environm export type MicrosoftRequest = Omit< Request, - 'user' | 'workspace' | 'cacheVersion' + 'user' | 'workspace' | 'workspaceMetadataVersion' > & { user: { firstName?: string | null; diff --git a/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts b/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts index f8d5f609d..4df605eba 100644 --- a/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts +++ b/packages/twenty-server/src/engine/core-modules/auth/types/google-api-request.type.ts @@ -5,7 +5,7 @@ import { MessageChannelVisibility } from 'src/modules/messaging/common/standard- export type GoogleAPIsRequest = Omit< Request, - 'user' | 'workspace' | 'cacheVersion' + 'user' | 'workspace' | 'workspaceMetadataVersion' > & { user: { firstName?: string | null; diff --git a/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts b/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts index beb53c4d0..2124c2b17 100644 --- a/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/file/controllers/file.controller.ts @@ -44,6 +44,10 @@ export class FileController { workspaceId, ); + fileStream.on('error', () => { + res.status(500).send({ error: 'Internal server error' }); + }); + fileStream.pipe(res); } catch (error) { if ( diff --git a/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts b/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts index ef47046fe..9785cde8a 100644 --- a/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts +++ b/packages/twenty-server/src/engine/core-modules/graphql/hooks/use-graphql-error-handler.hook.ts @@ -119,13 +119,13 @@ export const useGraphQLErrorHandlerHook = < if (Array.isArray(errors) && errors.length > 0) { const headers = context.req.headers; - const currentSchemaVersion = context.req.cacheVersion; + const currentMetadataVersion = context.req.workspaceMetadataVersion; - const requestSchemaVersion = headers['x-schema-version']; + const requestMetadataVersion = headers['x-schema-version']; if ( - requestSchemaVersion && - requestSchemaVersion !== currentSchemaVersion + requestMetadataVersion && + requestMetadataVersion !== `${currentMetadataVersion}` ) { throw new GraphQLError( `Schema version mismatch, please refresh the page.`, diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts index 1d5fb30c5..c77bb690b 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts @@ -33,7 +33,6 @@ import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorat import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; const getHMACKey = (email?: string, key?: string | null) => { @@ -54,7 +53,6 @@ export class UserResolver { private readonly environmentService: EnvironmentService, private readonly fileUploadService: FileUploadService, private readonly onboardingService: OnboardingService, - private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext, private readonly userVarService: UserVarsService, private readonly fileService: FileService, ) {} @@ -189,11 +187,6 @@ export class UserResolver { @ResolveField(() => OnboardingStatus) async onboardingStatus(@Parent() user: User): Promise { - const contextInstance = await this.loadServiceWithWorkspaceContext.load( - this.onboardingService, - user.defaultWorkspaceId, - ); - - return contextInstance.getOnboardingStatus(user); + return this.onboardingService.getOnboardingStatus(user); } } diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts index 0b4a3e048..b9948f623 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.entity.ts @@ -117,4 +117,16 @@ export class Workspace { (postgresCredentials) => postgresCredentials.workspace, ) allPostgresCredentials: Relation; + + @Field() + @Column({ default: 1 }) + metadataVersion: number; + + @Field() + @Column({ default: '' }) + databaseUrl: string; + + @Field() + @Column({ default: '' }) + databaseSchema: string; } diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index a8255d899..5c8e83266 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -16,7 +16,7 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { WorkspaceWorkspaceMemberListener } from 'src/engine/core-modules/workspace/workspace-workspace-member.listener'; import { WorkspaceResolver } from 'src/engine/core-modules/workspace/workspace.resolver'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts'; @@ -32,7 +32,7 @@ import { WorkspaceService } from './services/workspace.service'; BillingModule, FileModule, FileUploadModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, NestjsQueryTypeOrmModule.forFeature( [User, Workspace, UserWorkspace, FeatureFlagEntity], 'core', diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts index 1bb019344..bc995017b 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.resolver.ts @@ -26,7 +26,7 @@ import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator'; import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; import { DemoEnvGuard } from 'src/engine/guards/demo.env.guard'; import { JwtAuthGuard } from 'src/engine/guards/jwt.auth.guard'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { assert } from 'src/utils/assert'; import { streamToBuffer } from 'src/utils/stream-to-buffer'; @@ -39,7 +39,7 @@ import { WorkspaceService } from './services/workspace.service'; export class WorkspaceResolver { constructor( private readonly workspaceService: WorkspaceService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly userWorkspaceService: UserWorkspaceService, private readonly fileUploadService: FileUploadService, private readonly fileService: FileService, @@ -107,13 +107,6 @@ export class WorkspaceResolver { return this.workspaceService.deleteWorkspace(id); } - @ResolveField(() => String, { nullable: true }) - async currentCacheVersion( - @Parent() workspace: Workspace, - ): Promise { - return this.workspaceCacheVersionService.getVersion(workspace.id); - } - @ResolveField(() => BillingSubscription, { nullable: true }) async currentBillingSubscription( @Parent() workspace: Workspace, @@ -133,11 +126,15 @@ export class WorkspaceResolver { @ResolveField(() => String) async logo(@Parent() workspace: Workspace): Promise { if (workspace.logo) { - const workspaceLogoToken = await this.fileService.encodeFileToken({ - workspace_id: workspace.id, - }); + try { + const workspaceLogoToken = await this.fileService.encodeFileToken({ + workspace_id: workspace.id, + }); - return `${workspace.logo}?token=${workspaceLogoToken}`; + return `${workspace.logo}?token=${workspaceLogoToken}`; + } catch (e) { + return workspace.logo; + } } return workspace.logo ?? ''; diff --git a/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts b/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts index e89fb00b0..9296a629c 100644 --- a/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts +++ b/packages/twenty-server/src/engine/integrations/cache-storage/types/cache-storage-namespace.enum.ts @@ -1,5 +1,5 @@ export enum CacheStorageNamespace { - Messaging = 'messaging', - Calendar = 'calendar', - WorkspaceSchema = 'workspaceSchema', + ModuleMessaging = 'module:messaging', + ModuleCalendar = 'module:calendar', + EngineWorkspace = 'engine:workspace', } diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts index 7eac88744..c8797101d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts @@ -13,15 +13,15 @@ import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-s import { FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto'; import { FieldMetadataResolver } from 'src/engine/metadata-modules/field-metadata/field-metadata.resolver'; import { FieldMetadataGraphqlApiExceptionInterceptor } from 'src/engine/metadata-modules/field-metadata/interceptors/field-metadata-graphql-api-exception.interceptor'; +import { CreatedByPreQueryHook } from 'src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook'; import { IsFieldMetadataDefaultValue } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-default-value.validator'; import { IsFieldMetadataOptions } from 'src/engine/metadata-modules/field-metadata/validators/is-field-metadata-options.validator'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; import { WorkspaceStatusModule } from 'src/engine/workspace-manager/workspace-status/workspace-manager.module'; -import { CreatedByPreQueryHook } from 'src/engine/metadata-modules/field-metadata/query-hooks/created-by.pre-query-hook'; import { FieldMetadataEntity } from './field-metadata.entity'; import { FieldMetadataService } from './field-metadata.service'; @@ -38,7 +38,7 @@ import { UpdateFieldInput } from './dtos/update-field.input'; WorkspaceStatusModule, TwentyORMModule, WorkspaceMigrationRunnerModule, - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, ObjectMetadataModule, DataSourceModule, TypeORMModule, diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts index 9dfe1415c..8b26733dc 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.service.ts @@ -40,7 +40,7 @@ import { NameTooLongException, validateMetadataNameOrThrow, } from 'src/engine/metadata-modules/utils/validate-metadata-name.utils'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { generateMigrationName } from 'src/engine/metadata-modules/workspace-migration/utils/generate-migration-name.util'; import { WorkspaceMigrationColumnActionType, @@ -76,7 +76,7 @@ export class FieldMetadataService extends TypeOrmQueryService >, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly dataSourceService: DataSourceService, private readonly objectMetadataService: ObjectMetadataService, private readonly fieldMetadataService: FieldMetadataService, @@ -250,7 +250,9 @@ export class RemoteTableService { await this.remoteTableRepository.save(remoteTableEntity); - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); return { id: remoteTableEntity.id, @@ -435,7 +437,9 @@ export class RemoteTableService { await this.remoteTableRepository.delete(remoteTable.id); - await this.workspaceCacheVersionService.incrementVersion(workspaceId); + await this.workspaceMetadataVersionService.incrementMetadataVersion( + workspaceId, + ); } private async createRemoteTableMetadata( diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts deleted file mode 100644 index fab0a85d5..000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm'; - -@Entity('workspaceCacheVersion') -export class WorkspaceCacheVersionEntity { - @PrimaryGeneratedColumn('uuid') - id: string; - - @Column({ unique: true, nullable: false, type: 'uuid' }) - workspaceId: string; - - @Column() - version: string; - - @CreateDateColumn({ type: 'timestamptz' }) - createdAt: Date; - - @UpdateDateColumn({ type: 'timestamptz' }) - updatedAt: Date; -} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts deleted file mode 100644 index a0df038e4..000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; - -import { WorkspaceCacheVersionEntity } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; - -@Module({ - imports: [ - TypeOrmModule.forFeature([WorkspaceCacheVersionEntity], 'metadata'), - ], - exports: [WorkspaceCacheVersionService], - providers: [WorkspaceCacheVersionService], -}) -export class WorkspaceCacheVersionModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts deleted file mode 100644 index 8b0bfb8c2..000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; - -import { Repository } from 'typeorm'; - -import { WorkspaceCacheVersionEntity } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.entity'; - -@Injectable() -export class WorkspaceCacheVersionService { - constructor( - @InjectRepository(WorkspaceCacheVersionEntity, 'metadata') - private readonly workspaceCacheVersionRepository: Repository, - ) {} - - async incrementVersion(workspaceId: string): Promise { - const workspaceCacheVersion = (await this.getVersion(workspaceId)) ?? '0'; - const newVersion = `${+workspaceCacheVersion + 1}`; - - await this.workspaceCacheVersionRepository.upsert( - { - workspaceId, - version: `${+workspaceCacheVersion + 1}`, - }, - ['workspaceId'], - ); - - return newVersion; - } - - async getVersion(workspaceId: string): Promise { - const workspaceCacheVersion = - await this.workspaceCacheVersionRepository.findOne({ - where: { workspaceId }, - }); - - return workspaceCacheVersion?.version ?? null; - } - - async deleteVersion(workspaceId: string): Promise { - await this.workspaceCacheVersionRepository.delete({ workspaceId }); - } -} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts new file mode 100644 index 000000000..4064872a1 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; +import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; + +@Module({ + imports: [ + TypeOrmModule.forFeature([Workspace], 'core'), + WorkspaceCacheStorageModule, + ], + exports: [WorkspaceMetadataVersionService], + providers: [WorkspaceMetadataVersionService], +}) +export class WorkspaceMetadataVersionModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts new file mode 100644 index 000000000..8e5c5deaa --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service.ts @@ -0,0 +1,79 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; + +@Injectable() +export class WorkspaceMetadataVersionService { + logger = new Logger(WorkspaceMetadataVersionService.name); + + constructor( + @InjectRepository(Workspace, 'core') + private readonly workspaceRepository: Repository, + private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + ) {} + + async flushCacheIfMetadataVersionIsOutdated( + workspaceId: string, + ): Promise { + const currentVersion = + (await this.workspaceCacheStorageService.getMetadataVersion( + workspaceId, + )) ?? 1; + + let latestVersion = await this.getMetadataVersion(workspaceId); + + if (latestVersion === undefined || currentVersion !== latestVersion) { + this.logger.log( + `Metadata version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`, + ); + + await this.workspaceCacheStorageService.flush(workspaceId); + + latestVersion = await this.incrementMetadataVersion(workspaceId); + + await this.workspaceCacheStorageService.setMetadataVersion( + workspaceId, + latestVersion, + ); + } + } + + async incrementMetadataVersion(workspaceId: string): Promise { + const metadataVersion = (await this.getMetadataVersion(workspaceId)) ?? 0; + const newMetadataVersion = metadataVersion + 1; + + await this.workspaceRepository.update( + { id: workspaceId }, + { metadataVersion: newMetadataVersion }, + ); + + await this.workspaceCacheStorageService.setMetadataVersion( + workspaceId, + newMetadataVersion, + ); + + return newMetadataVersion; + } + + async getMetadataVersion(workspaceId: string): Promise { + const workspace = await this.workspaceRepository.findOne({ + where: { id: workspaceId }, + }); + + return workspace?.metadataVersion; + } + + async resetMetadataVersion(workspaceId: string): Promise { + await this.workspaceRepository.update( + { id: workspaceId }, + { metadataVersion: 1 }, + ); + + await this.workspaceCacheStorageService.flush(workspaceId); + await this.workspaceCacheStorageService.setMetadataVersion(workspaceId, 1); + } +} diff --git a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts index a756b8cef..509e58bb3 100644 --- a/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts +++ b/packages/twenty-server/src/engine/middlewares/graphql-hydrate-request-from-token.middleware.ts @@ -6,7 +6,7 @@ import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filt import { TokenService } from 'src/engine/core-modules/auth/services/token.service'; import { AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { ExceptionHandlerService } from 'src/engine/integrations/exception-handler/exception-handler.service'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { handleExceptionAndConvertToGraphQLError } from 'src/engine/utils/global-exception-handler.util'; class GraphqlTokenValidationProxy { @@ -33,7 +33,7 @@ export class GraphQLHydrateRequestFromTokenMiddleware { constructor( private readonly tokenService: TokenService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, private readonly exceptionHandlerService: ExceptionHandlerService, ) {} @@ -72,15 +72,16 @@ export class GraphQLHydrateRequestFromTokenMiddleware ); data = await graphqlTokenValidationProxy.validateToken(req); - const cacheVersion = await this.workspaceCacheVersionService.getVersion( - data.workspace.id, - ); + const metadataVersion = + await this.workspaceMetadataVersionService.getMetadataVersion( + data.workspace.id, + ); req.user = data.user; req.apiKey = data.apiKey; req.workspace = data.workspace; req.workspaceId = data.workspace.id; - req.cacheVersion = cacheVersion; + req.workspaceMetadataVersion = metadataVersion; } catch (error) { res.writeHead(200, { 'Content-Type': 'application/json' }); res.write( diff --git a/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts b/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts deleted file mode 100644 index 144f47598..000000000 --- a/packages/twenty-server/src/engine/twenty-orm/context/load-service-with-workspace.context.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Inject, Type } from '@nestjs/common'; -import { ModuleRef, createContextId } from '@nestjs/core'; -import { Injector } from '@nestjs/core/injector/injector'; - -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; - -export class LoadServiceWithWorkspaceContext { - private readonly injector = new Injector(); - - constructor( - @Inject(ModuleRef) - private readonly moduleRef: ModuleRef, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, - ) {} - - async load(service: T, workspaceId: string): Promise { - const modules = this.moduleRef['container'].getModules(); - const host = [...modules.values()].find((module) => - module.providers.has((service as Type).constructor), - ); - - if (!host) { - throw new Error('Host module not found for the service'); - } - - const contextId = createContextId(); - const cacheVersion = - await this.workspaceCacheVersionService.getVersion(workspaceId); - - if (this.moduleRef.registerRequestByContextId) { - this.moduleRef.registerRequestByContextId( - { req: { workspaceId, cacheVersion } }, - contextId, - ); - } - - return this.injector.loadPerContext( - service, - host, - new Map(host.providers), - contextId, - ); - } -} diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts index 88603807a..bf70156a9 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/scoped-workspace-context.factory.ts @@ -11,16 +11,16 @@ export class ScopedWorkspaceContextFactory { public create(): { workspaceId: string | null; - cacheVersion: string | null; + workspaceMetadataVersion: string | null; } { const workspaceId: string | undefined = this.request?.['req']?.['workspaceId']; - const cacheVersion: string | undefined = - this.request?.['req']?.['cacheVersion']; + const workspaceMetadataVersion: string | undefined = + this.request?.['req']?.['workspaceMetadataVersion']; return { workspaceId: workspaceId ?? null, - cacheVersion: cacheVersion ?? null, + workspaceMetadataVersion: workspaceMetadataVersion ?? null, }; } } diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 1ee50a0b4..10daedc7d 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -1,12 +1,12 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { Repository } from 'typeorm'; +import { EntitySchema, Repository } from 'typeorm'; import { EnvironmentService } from 'src/engine/integrations/environment/environment.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { workspaceDataSourceCacheInstance } from 'src/engine/twenty-orm/twenty-orm-core.module'; @@ -18,7 +18,7 @@ export class WorkspaceDatasourceFactory { private readonly dataSourceService: DataSourceService, private readonly environmentService: EnvironmentService, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, @InjectRepository(ObjectMetadataEntity, 'metadata') private readonly objectMetadataRepository: Repository, private readonly entitySchemaFactory: EntitySchemaFactory, @@ -26,21 +26,29 @@ export class WorkspaceDatasourceFactory { public async create( workspaceId: string, - workspaceSchemaVersion: string | null, + workspaceMetadataVersion: string | null, ): Promise { - const desiredWorkspaceSchemaVersion = - workspaceSchemaVersion ?? - (await this.workspaceCacheVersionService.getVersion(workspaceId)); + const desiredWorkspaceMetadataVersion = + workspaceMetadataVersion ?? + (await this.workspaceMetadataVersionService.getMetadataVersion( + workspaceId, + )); - if (!desiredWorkspaceSchemaVersion) { - throw new Error('Cache version not found'); + if (!desiredWorkspaceMetadataVersion) { + throw new Error( + `Desired workspace metadata version not found while creating workspace data source for workspace ${workspaceId}`, + ); } - const latestWorkspaceSchemaVersion = - await this.workspaceCacheVersionService.getVersion(workspaceId); + const latestWorkspaceMetadataVersion = + await this.workspaceMetadataVersionService.getMetadataVersion( + workspaceId, + ); - if (latestWorkspaceSchemaVersion !== desiredWorkspaceSchemaVersion) { - throw new Error('Cache version mismatch'); + if (latestWorkspaceMetadataVersion !== desiredWorkspaceMetadataVersion) { + throw new Error( + `Workspace metadata version mismatch detected for workspace ${workspaceId}. Current version: ${latestWorkspaceMetadataVersion}. Desired version: ${desiredWorkspaceMetadataVersion}`, + ); } let cachedObjectMetadataCollection = @@ -70,7 +78,7 @@ export class WorkspaceDatasourceFactory { } const workspaceDataSource = await workspaceDataSourceCacheInstance.execute( - `${workspaceId}-${latestWorkspaceSchemaVersion}`, + `${workspaceId}-${latestWorkspaceMetadataVersion}`, async () => { const dataSourceMetadata = await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId( @@ -78,18 +86,43 @@ export class WorkspaceDatasourceFactory { ); if (!dataSourceMetadata) { - throw new Error('Data source metadata not found'); + throw new Error( + `Data source metadata not found for workspace ${workspaceId}`, + ); } if (!cachedObjectMetadataCollection) { - throw new Error('Object metadata collection not found'); + throw new Error( + `Object metadata collection not found for workspace ${workspaceId}`, + ); + } + + const cachedEntitySchemaOptions = + await this.workspaceCacheStorageService.getORMEntitySchema( + workspaceId, + ); + + let cachedEntitySchemas: EntitySchema[]; + + if (cachedEntitySchemaOptions) { + cachedEntitySchemas = cachedEntitySchemaOptions.map( + (option) => new EntitySchema(option), + ); + } else { + const entitySchemas = await Promise.all( + cachedObjectMetadataCollection.map((objectMetadata) => + this.entitySchemaFactory.create(workspaceId, objectMetadata), + ), + ); + + await this.workspaceCacheStorageService.setORMEntitySchema( + workspaceId, + entitySchemas.map((entitySchema) => entitySchema.options), + ); + + cachedEntitySchemas = entitySchemas; } - const entities = await Promise.all( - cachedObjectMetadataCollection.map((objectMetadata) => - this.entitySchemaFactory.create(workspaceId, objectMetadata), - ), - ); const workspaceDataSource = new WorkspaceDataSource( { workspaceId, @@ -104,7 +137,7 @@ export class WorkspaceDatasourceFactory { ? ['query', 'error'] : ['error'], schema: dataSourceMetadata.schema, - entities, + entities: cachedEntitySchemas, ssl: this.environmentService.get('PG_SSL_ALLOW_SELF_SIGNED') ? { rejectUnauthorized: false, diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts index ebb929a64..122ae0c52 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-core.module.ts @@ -14,8 +14,7 @@ import { import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceDataSource } from 'src/engine/twenty-orm/datasource/workspace.datasource'; import { entitySchemaFactories } from 'src/engine/twenty-orm/factories'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; @@ -33,21 +32,15 @@ export const workspaceDataSourceCacheInstance = imports: [ TypeOrmModule.forFeature([ObjectMetadataEntity], 'metadata'), DataSourceModule, - WorkspaceCacheVersionModule, WorkspaceCacheStorageModule, + WorkspaceMetadataVersionModule, ], providers: [ ...entitySchemaFactories, TwentyORMManager, TwentyORMGlobalManager, - LoadServiceWithWorkspaceContext, - ], - exports: [ - EntitySchemaFactory, - TwentyORMManager, - LoadServiceWithWorkspaceContext, - TwentyORMGlobalManager, ], + exports: [EntitySchemaFactory, TwentyORMManager, TwentyORMGlobalManager], }) export class TwentyORMCoreModule extends ConfigurableModuleClass diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts index 5ebd65a3c..6855c7056 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.manager.ts @@ -25,7 +25,7 @@ export class TwentyORMManager { async getRepository( workspaceEntityOrobjectMetadataName: Type | string, ): Promise> { - const { workspaceId, cacheVersion } = + const { workspaceId, workspaceMetadataVersion } = this.scopedWorkspaceContextFactory.create(); let objectMetadataName: string; @@ -44,20 +44,23 @@ export class TwentyORMManager { const workspaceDataSource = await this.workspaceDataSourceFactory.create( workspaceId, - cacheVersion, + workspaceMetadataVersion, ); return workspaceDataSource.getRepository(objectMetadataName); } async getDatasource() { - const { workspaceId, cacheVersion } = + const { workspaceId, workspaceMetadataVersion } = this.scopedWorkspaceContextFactory.create(); if (!workspaceId) { throw new Error('Workspace not found'); } - return this.workspaceDataSourceFactory.create(workspaceId, cacheVersion); + return this.workspaceDataSourceFactory.create( + workspaceId, + workspaceMetadataVersion, + ); } } diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts index 62d8be6fd..fec521528 100644 --- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts +++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.module.ts @@ -1,10 +1,8 @@ import { Module } from '@nestjs/common'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @Module({ - imports: [WorkspaceCacheVersionModule], providers: [WorkspaceCacheStorageService], exports: [WorkspaceCacheStorageService], }) diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts index 616084702..18426da5a 100644 --- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts +++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts @@ -1,56 +1,67 @@ import { Injectable, Logger } from '@nestjs/common'; +import { EntitySchemaOptions } from 'typeorm'; + import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; + +enum WorkspaceCacheKeys { + GraphQLTypeDefs = 'graphql:type-defs', + GraphQLUsedScalarNames = 'graphql:used-scalar-names', + GraphQLOperations = 'graphql:operations', + ORMEntitySchemas = 'orm:entity-schemas', + MetadataObjectMetadataCollection = 'metadata:object-metadata-collection', + MetadataVersion = 'metadata:workspace-metadata-version', +} @Injectable() export class WorkspaceCacheStorageService { private readonly logger = new Logger(WorkspaceCacheStorageService.name); constructor( - @InjectCacheStorage(CacheStorageNamespace.WorkspaceSchema) - private readonly workspaceSchemaCache: CacheStorageService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, + @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) + private readonly cacheStorageService: CacheStorageService, ) {} - async validateCacheVersion(workspaceId: string): Promise { - const currentVersion = - (await this.workspaceSchemaCache.get( - `cacheVersion:${workspaceId}`, - )) ?? '0'; + setORMEntitySchema( + workspaceId: string, + entitySchemas: EntitySchemaOptions[], + ) { + return this.cacheStorageService.set[]>( + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, + entitySchemas, + ); + } - let latestVersion = - await this.workspaceCacheVersionService.getVersion(workspaceId); + getORMEntitySchema( + workspaceId: string, + ): Promise[] | undefined> { + return this.cacheStorageService.get[]>( + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, + ); + } - if (!latestVersion || currentVersion !== latestVersion) { - // Invalidate cache if version mismatch is detected" - this.logger.log( - `Cache version mismatch detected for workspace ${workspaceId}. Current version: ${currentVersion}. Latest version: ${latestVersion}. Invalidating cache...`, - ); + setMetadataVersion(workspaceId: string, version: number): Promise { + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, + version, + ); + } - await this.invalidateCache(workspaceId); - - // If the latest version is not found, increment the version - latestVersion ??= - await this.workspaceCacheVersionService.incrementVersion(workspaceId); - - // Update the cache version after invalidation - await this.workspaceSchemaCache.set( - `cacheVersion:${workspaceId}`, - latestVersion, - ); - } + getMetadataVersion(workspaceId: string): Promise { + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, + ); } setObjectMetadataCollection( workspaceId: string, objectMetadataCollection: ObjectMetadataEntity[], ) { - return this.workspaceSchemaCache.set( - `objectMetadataCollection:${workspaceId}`, + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, objectMetadataCollection, ); } @@ -58,43 +69,57 @@ export class WorkspaceCacheStorageService { getObjectMetadataCollection( workspaceId: string, ): Promise { - return this.workspaceSchemaCache.get( - `objectMetadataCollection:${workspaceId}`, + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, ); } - setTypeDefs(workspaceId: string, typeDefs: string): Promise { - return this.workspaceSchemaCache.set( - `typeDefs:${workspaceId}`, + setGraphQLTypeDefs(workspaceId: string, typeDefs: string): Promise { + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, typeDefs, ); } - getTypeDefs(workspaceId: string): Promise { - return this.workspaceSchemaCache.get(`typeDefs:${workspaceId}`); + getGraphQLTypeDefs(workspaceId: string): Promise { + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, + ); } - setUsedScalarNames( + setGraphQLUsedScalarNames( workspaceId: string, - scalarsUsed: string[], + usedScalarNames: string[], ): Promise { - return this.workspaceSchemaCache.set( - `usedScalarNames:${workspaceId}`, - scalarsUsed, + return this.cacheStorageService.set( + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, + usedScalarNames, ); } - getUsedScalarNames(workspaceId: string): Promise { - return this.workspaceSchemaCache.get( - `usedScalarNames:${workspaceId}`, + getGraphQLUsedScalarNames( + workspaceId: string, + ): Promise { + return this.cacheStorageService.get( + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, ); } - async invalidateCache(workspaceId: string): Promise { - await this.workspaceSchemaCache.del( - `objectMetadataCollection:${workspaceId}`, + async flush(workspaceId: string): Promise { + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.MetadataObjectMetadataCollection}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.MetadataVersion}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.GraphQLTypeDefs}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.GraphQLUsedScalarNames}:${workspaceId}`, + ); + await this.cacheStorageService.del( + `${WorkspaceCacheKeys.ORMEntitySchemas}:${workspaceId}`, ); - await this.workspaceSchemaCache.del(`typeDefs:${workspaceId}`); - await this.workspaceSchemaCache.del(`usedScalarNames:${workspaceId}`); } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts index 2bbc17b75..ffa4c387d 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-cleaner/commands/delete-workspaces.command.ts @@ -1,14 +1,13 @@ -import { InjectRepository } from '@nestjs/typeorm'; import { Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; import { Command, CommandRunner, Option } from 'nest-commander'; import { In, Repository } from 'typeorm'; +import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { getDryRunLogHeader } from 'src/utils/get-dry-run-log-header'; -import { WorkspaceService } from 'src/engine/core-modules/workspace/services/workspace.service'; -import { LoadServiceWithWorkspaceContext } from 'src/engine/twenty-orm/context/load-service-with-workspace.context'; type DeleteWorkspacesCommandOptions = { dryRun?: boolean; @@ -24,7 +23,6 @@ export class DeleteWorkspacesCommand extends CommandRunner { constructor( private readonly workspaceService: WorkspaceService, - private readonly loadServiceWithWorkspaceContext: LoadServiceWithWorkspaceContext, @InjectRepository(Workspace, 'core') private readonly workspaceRepository: Repository, private readonly dataSourceService: DataSourceService, @@ -81,15 +79,15 @@ export class DeleteWorkspacesCommand extends CommandRunner { workspace.id } name: '${workspace.displayName}'`, ); - const workspaceServiceInstance = - await this.loadServiceWithWorkspaceContext.load( - this.workspaceService, - workspace.id, - ); + // const workspaceServiceInstance = + // await this.loadServiceWithWorkspaceContext.load( + // this.workspaceService, + // workspace.id, + // ); - if (!options.dryRun) { - await workspaceServiceInstance.softDeleteWorkspace(workspace.id); - } + // if (!options.dryRun) { + // await workspaceServiceInstance.softDeleteWorkspace(workspace.id); + // } } } } diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts index 59f0bec2e..40f6e5d92 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module.ts @@ -2,7 +2,6 @@ import { Module } from '@nestjs/common'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; import { WorkspaceMigrationEnumService } from 'src/engine/workspace-manager/workspace-migration-runner/services/workspace-migration-enum.service'; import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.service'; @@ -10,11 +9,7 @@ import { WorkspaceMigrationRunnerService } from './workspace-migration-runner.se import { WorkspaceMigrationTypeService } from './services/workspace-migration-type.service'; @Module({ - imports: [ - WorkspaceDataSourceModule, - WorkspaceMigrationModule, - WorkspaceCacheVersionModule, - ], + imports: [WorkspaceDataSourceModule, WorkspaceMigrationModule], providers: [ WorkspaceMigrationRunnerService, WorkspaceMigrationEnumService, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts index 447f60e0c..76cd7bfe9 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.module.ts @@ -6,7 +6,7 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature- import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { RelationMetadataEntity } from 'src/engine/metadata-modules/relation-metadata/relation-metadata.entity'; -import { WorkspaceCacheVersionModule } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationBuilderModule } from 'src/engine/workspace-manager/workspace-migration-builder/workspace-migration-builder.module'; import { WorkspaceMigrationRunnerModule } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.module'; @@ -35,7 +35,7 @@ import { WorkspaceSyncMetadataService } from 'src/engine/workspace-manager/works 'metadata', ), TypeOrmModule.forFeature([FeatureFlagEntity], 'core'), - WorkspaceCacheVersionModule, + WorkspaceMetadataVersionModule, ], providers: [ ...workspaceSyncMetadataFactories, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts index 2a22bac59..6dfeb5557 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/workspace-sync-metadata.service.ts @@ -6,7 +6,7 @@ import { DataSource, QueryFailedError } from 'typeorm'; import { WorkspaceSyncContext } from 'src/engine/workspace-manager/workspace-sync-metadata/interfaces/workspace-sync-context.interface'; import { FeatureFlagFactory } from 'src/engine/core-modules/feature-flag/services/feature-flags.factory'; -import { WorkspaceCacheVersionService } from 'src/engine/metadata-modules/workspace-cache-version/workspace-cache-version.service'; +import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.service'; import { WorkspaceMigrationEntity } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.entity'; import { WorkspaceMigrationRunnerService } from 'src/engine/workspace-manager/workspace-migration-runner/workspace-migration-runner.service'; import { WorkspaceSyncFieldMetadataService } from 'src/engine/workspace-manager/workspace-sync-metadata/services/workspace-sync-field-metadata.service'; @@ -32,9 +32,9 @@ export class WorkspaceSyncMetadataService { private readonly workspaceSyncObjectMetadataService: WorkspaceSyncObjectMetadataService, private readonly workspaceSyncRelationMetadataService: WorkspaceSyncRelationMetadataService, private readonly workspaceSyncFieldMetadataService: WorkspaceSyncFieldMetadataService, - private readonly workspaceCacheVersionService: WorkspaceCacheVersionService, private readonly workspaceSyncIndexMetadataService: WorkspaceSyncIndexMetadataService, private readonly workspaceSyncObjectMetadataIdentifiersService: WorkspaceSyncObjectMetadataIdentifiersService, + private readonly workspaceMetadataVersionService: WorkspaceMetadataVersionService, ) {} /** @@ -157,7 +157,7 @@ export class WorkspaceSyncMetadataService { await queryRunner.rollbackTransaction(); } finally { await queryRunner.release(); - await this.workspaceCacheVersionService.incrementVersion( + await this.workspaceMetadataVersionService.incrementMetadataVersion( context.workspaceId, ); } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts index 87d58c3f6..56f9af210 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/crons/jobs/calendar-event-list-fetch.cron.job.ts @@ -32,6 +32,8 @@ export class CalendarEventListFetchCronJob { @Process(CalendarEventListFetchCronJob.name) async handle(): Promise { + console.time('CalendarEventListFetchCronJob time'); + const activeWorkspaces = await this.workspaceRepository.find({ where: { activationStatus: WorkspaceActivationStatus.ACTIVE, @@ -65,5 +67,7 @@ export class CalendarEventListFetchCronJob { ); } } + + console.timeEnd('CalendarEventListFetchCronJob time'); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts index 9006872ac..402661b93 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job.ts @@ -33,6 +33,8 @@ export class CalendarEventListFetchJob { @Process(CalendarEventListFetchJob.name) async handle(data: CalendarEventsImportJobData): Promise { + console.time('CalendarEventListFetchJob time'); + const { workspaceId, calendarChannelId } = data; const calendarChannelRepository = @@ -91,5 +93,6 @@ export class CalendarEventListFetchJob { default: break; } + console.timeEnd('CalendarEventListFetchJob time'); } } diff --git a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts index 28306cea8..21ccdaafc 100644 --- a/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/calendar/calendar-event-import-manager/services/calendar-channel-sync-status.service.ts @@ -16,7 +16,7 @@ import { AccountsToReconnectKeys } from 'src/modules/connected-account/types/acc export class CalendarChannelSyncStatusService { constructor( private readonly twentyORMManager: TwentyORMManager, - @InjectCacheStorage(CacheStorageNamespace.Calendar) + @InjectCacheStorage(CacheStorageNamespace.ModuleCalendar) private readonly cacheStorage: CacheStorageService, private readonly accountsToReconnectService: AccountsToReconnectService, ) {} diff --git a/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts b/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts index 49c52cb83..43025a021 100644 --- a/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts +++ b/packages/twenty-server/src/modules/messaging/common/services/messaging-channel-sync-status.service.ts @@ -15,7 +15,7 @@ import { @Injectable() export class MessagingChannelSyncStatusService { constructor( - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly twentyORMManager: TwentyORMManager, private readonly accountsToReconnectService: AccountsToReconnectService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts index 3f7334cf7..2f4641882 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-message-list-fetch.cron.job.ts @@ -33,6 +33,8 @@ export class MessagingMessageListFetchCronJob { @Process(MessagingMessageListFetchCronJob.name) async handle(): Promise { + console.time('MessagingMessageListFetchCronJob time'); + const activeWorkspaces = await this.workspaceRepository.find({ where: { activationStatus: WorkspaceActivationStatus.ACTIVE, @@ -66,5 +68,7 @@ export class MessagingMessageListFetchCronJob { } } } + + console.timeEnd('MessagingMessageListFetchCronJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts index e3da8b05b..daef69dfa 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/crons/jobs/messaging-messages-import.cron.job.ts @@ -36,6 +36,7 @@ export class MessagingMessagesImportCronJob { @Process(MessagingMessagesImportCronJob.name) async handle(): Promise { + console.time('MessagingMessagesImportCronJob time'); const activeWorkspaces = await this.workspaceRepository.find({ where: { activationStatus: WorkspaceActivationStatus.ACTIVE, @@ -69,5 +70,7 @@ export class MessagingMessagesImportCronJob { } } } + + console.timeEnd('MessagingMessagesImportCronJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts index d81bba9cd..3f2c2a537 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-add-single-message-to-cache-for-import.job.ts @@ -14,7 +14,7 @@ export type MessagingAddSingleMessageToCacheForImportJobData = { @Processor(MessageQueue.messagingQueue) export class MessagingAddSingleMessageToCacheForImportJob { constructor( - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, ) {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts index 8c2bc681f..24e048ce0 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-clean-cache.ts @@ -1,11 +1,11 @@ import { Logger } from '@nestjs/common'; -import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; -import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; -import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; +import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; import { InjectCacheStorage } from 'src/engine/integrations/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageNamespace } from 'src/engine/integrations/cache-storage/types/cache-storage-namespace.enum'; -import { CacheStorageService } from 'src/engine/integrations/cache-storage/cache-storage.service'; +import { Process } from 'src/engine/integrations/message-queue/decorators/process.decorator'; +import { Processor } from 'src/engine/integrations/message-queue/decorators/processor.decorator'; +import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; export type MessagingCleanCacheJobData = { workspaceId: string; @@ -17,7 +17,7 @@ export class MessagingCleanCacheJob { private readonly logger = new Logger(MessagingCleanCacheJob.name); constructor( - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, ) {} diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts index 09854fbf4..fc0d37413 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-message-list-fetch.job.ts @@ -39,6 +39,8 @@ export class MessagingMessageListFetchJob { @Process(MessagingMessageListFetchJob.name) async handle(data: MessagingMessageListFetchJobData): Promise { + console.time('MessagingMessageListFetchJob time'); + const { messageChannelId, workspaceId } = data; await this.messagingTelemetryService.track({ @@ -145,5 +147,7 @@ export class MessagingMessageListFetchJob { default: break; } + + console.timeEnd('MessagingMessageListFetchJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts index 935be741e..c35243f8e 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/jobs/messaging-messages-import.job.ts @@ -35,6 +35,8 @@ export class MessagingMessagesImportJob { @Process(MessagingMessagesImportJob.name) async handle(data: MessagingMessagesImportJobData): Promise { + console.time('MessagingMessagesImportJob time'); + const { messageChannelId, workspaceId } = data; await this.messagingTelemetryService.track({ @@ -95,5 +97,7 @@ export class MessagingMessagesImportJob { connectedAccount, workspaceId, ); + + console.timeEnd('MessagingMessagesImportJob time'); } } diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts index dde37698e..eca0aaedf 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-full-message-list-fetch.service.ts @@ -29,7 +29,7 @@ export class MessagingFullMessageListFetchService { constructor( private readonly gmailClientProvider: MessagingGmailClientProvider, - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService, private readonly gmailErrorHandlingService: MessagingErrorHandlingService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts index 89fcedb6d..968140bc1 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts @@ -31,7 +31,7 @@ export class MessagingMessagesImportService { constructor( private readonly fetchMessagesByBatchesService: MessagingGmailFetchMessagesByBatchesService, - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService, private readonly saveMessagesAndEnqueueContactCreationService: MessagingSaveMessagesAndEnqueueContactCreationService, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts index 7ab5f5c65..05189ded1 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-partial-message-list-fetch.service.ts @@ -24,7 +24,7 @@ export class MessagingPartialMessageListFetchService { constructor( private readonly gmailClientProvider: MessagingGmailClientProvider, - @InjectCacheStorage(CacheStorageNamespace.Messaging) + @InjectCacheStorage(CacheStorageNamespace.ModuleMessaging) private readonly cacheStorage: CacheStorageService, private readonly gmailErrorHandlingService: MessagingErrorHandlingService, private readonly gmailGetHistoryService: MessagingGmailHistoryService, diff --git a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts index e3a3b5e06..94ac094ff 100644 --- a/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts +++ b/packages/twenty-server/src/modules/messaging/monitoring/crons/jobs/messaging-message-channel-sync-status-monitoring.cron.ts @@ -32,6 +32,8 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob { async handle(): Promise { this.logger.log('Starting message channel sync status monitoring...'); + console.time('MessagingMessageChannelSyncStatusMonitoringCronJob time'); + await this.messagingTelemetryService.track({ eventName: 'message_channel.monitoring.sync_status.start', message: 'Starting message channel sync status monitoring', @@ -68,5 +70,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob { }); } } + + console.timeEnd('MessagingMessageChannelSyncStatusMonitoringCronJob time'); } }