admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor + new account sync metrics + isolating health status inside folder admin-panel > health-status (#10314)

Browse Source 
closes https://github.com/twentyhq/core-team-issues/issues/444
https://github.com/twentyhq/core-team-issues/issues/443
https://github.com/twentyhq/core-team-issues/issues/442
This commit is contained in:
nitin
2025-02-21 14:18:47 +05:30
committed by GitHub
parent 41bbb4b47f
commit c46f7848b7
57 changed files with 1441 additions and 833 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
packages/twenty-front/src/generated/graphql.tsx
View File

@ -27,7 +27,10 @@ export type ActivateWorkspaceInput = {
export type AdminPanelHealthServiceData = {
__typename?: 'AdminPanelHealthServiceData';
description: Scalars['String'];
details?: Maybe<Scalars['String']>;
id: Scalars['String'];
label: Scalars['String'];
queues?: Maybe<Array<AdminPanelWorkerQueueHealth>>;
status: AdminPanelHealthServiceStatus;
};
@ -37,17 +40,11 @@ export enum AdminPanelHealthServiceStatus {
OUTAGE = 'OUTAGE'
}
export enum AdminPanelIndicatorHealthStatusInputEnum {
DATABASE = 'DATABASE',
MESSAGE_SYNC = 'MESSAGE_SYNC',
REDIS = 'REDIS',
WORKER = 'WORKER'
}
export type AdminPanelWorkerQueueHealth = {
__typename?: 'AdminPanelWorkerQueueHealth';
id: Scalars['String'];
metrics: WorkerQueueMetrics;
name: Scalars['String'];
queueName: Scalars['String'];
status: AdminPanelHealthServiceStatus;
workers: Scalars['Float'];
};
@ -614,6 +611,13 @@ export type GetServerlessFunctionSourceCodeInput = {
version?: Scalars['String'];
};
export enum HealthIndicatorId {
connectedAccount = 'connectedAccount',
database = 'database',
redis = 'redis',
worker = 'worker'
}
export enum IdentityProviderType {
OIDC = 'OIDC',
SAML = 'SAML'
@ -1306,7 +1310,7 @@ export type QueryGetAvailablePackagesArgs = {
export type QueryGetIndicatorHealthStatusArgs = {
indicatorName: AdminPanelIndicatorHealthStatusInputEnum;
indicatorId: HealthIndicatorId;
};
@ -1630,10 +1634,14 @@ export type Support = {
export type SystemHealth = {
__typename?: 'SystemHealth';
database: AdminPanelHealthServiceData;
messageSync: AdminPanelHealthServiceData;
redis: AdminPanelHealthServiceData;
worker: AdminPanelHealthServiceData;
services: Array<SystemHealthService>;
};
export type SystemHealthService = {
__typename?: 'SystemHealthService';
id: HealthIndicatorId;
label: Scalars['String'];
status: AdminPanelHealthServiceStatus;
};
export type TimelineCalendarEvent = {
@ -2292,16 +2300,16 @@ export type GetEnvironmentVariablesGroupedQueryVariables = Exact<{ [key: string]
export type GetEnvironmentVariablesGroupedQuery = { __typename?: 'Query', getEnvironmentVariablesGrouped: { __typename?: 'EnvironmentVariablesOutput', groups: Array<{ __typename?: 'EnvironmentVariablesGroupData', name: EnvironmentVariablesGroup, description: string, isHiddenOnLoad: boolean, variables: Array<{ __typename?: 'EnvironmentVariable', name: string, description: string, value: string, sensitive: boolean }> }> } };
export type GetIndicatorHealthStatusQueryVariables = Exact<{
indicatorName: AdminPanelIndicatorHealthStatusInputEnum;
indicatorId: HealthIndicatorId;
}>;
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', name: string, status: AdminPanelHealthServiceStatus, workers: number, metrics: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, prioritized: number } }> | null } };
export type GetIndicatorHealthStatusQuery = { __typename?: 'Query', getIndicatorHealthStatus: { __typename?: 'AdminPanelHealthServiceData', id: string, label: string, description: string, status: AdminPanelHealthServiceStatus, details?: string | null, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', id: string, queueName: string, status: AdminPanelHealthServiceStatus, workers: number, metrics: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, prioritized: number } }> | null } };
export type GetSystemHealthStatusQueryVariables = Exact<{ [key: string]: never; }>;
export type GetSystemHealthStatusQuery = { __typename?: 'Query', getSystemHealthStatus: { __typename?: 'SystemHealth', database: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null }, redis: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null }, worker: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, queues?: Array<{ __typename?: 'AdminPanelWorkerQueueHealth', name: string, workers: number, status: AdminPanelHealthServiceStatus, metrics: { __typename?: 'WorkerQueueMetrics', failed: number, completed: number, waiting: number, active: number, delayed: number, prioritized: number } }> | null }, messageSync: { __typename?: 'AdminPanelHealthServiceData', status: AdminPanelHealthServiceStatus, details?: string | null } } };
export type GetSystemHealthStatusQuery = { __typename?: 'Query', getSystemHealthStatus: { __typename?: 'SystemHealth', services: Array<{ __typename?: 'SystemHealthService', id: HealthIndicatorId, label: string, status: AdminPanelHealthServiceStatus }> } };
export type UpdateLabPublicFeatureFlagMutationVariables = Exact<{
input: UpdateLabPublicFeatureFlagInput;
@ -4018,12 +4026,16 @@ export type GetEnvironmentVariablesGroupedQueryHookResult = ReturnType<typeof us
export type GetEnvironmentVariablesGroupedLazyQueryHookResult = ReturnType<typeof useGetEnvironmentVariablesGroupedLazyQuery>;
export type GetEnvironmentVariablesGroupedQueryResult = Apollo.QueryResult<GetEnvironmentVariablesGroupedQuery, GetEnvironmentVariablesGroupedQueryVariables>;
export const GetIndicatorHealthStatusDocument = gql`
query GetIndicatorHealthStatus($indicatorName: AdminPanelIndicatorHealthStatusInputEnum!) {
getIndicatorHealthStatus(indicatorName: $indicatorName) {
query GetIndicatorHealthStatus($indicatorId: HealthIndicatorId!) {
getIndicatorHealthStatus(indicatorId: $indicatorId) {
id
label
description
status
details
queues {
name
id
queueName
status
workers
metrics {
@ -4051,7 +4063,7 @@ export const GetIndicatorHealthStatusDocument = gql`
* @example
* const { data, loading, error } = useGetIndicatorHealthStatusQuery({
* variables: {
* indicatorName: // value for 'indicatorName'
* indicatorId: // value for 'indicatorId'
* },
* });
*/
@ -4069,33 +4081,10 @@ export type GetIndicatorHealthStatusQueryResult = Apollo.QueryResult<GetIndicato
export const GetSystemHealthStatusDocument = gql`
query GetSystemHealthStatus {
getSystemHealthStatus {
database {
services {
id
label
status
details
}
redis {
status
details
}
worker {
status
queues {
name
workers
status
metrics {
failed
completed
waiting
active
delayed
prioritized
}
}
}
messageSync {
status
details
}
}
}

44
packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable.tsx
View File

@ -1,44 +0,0 @@
import { Table } from '@/ui/layout/table/components/Table';
import { TableCell } from '@/ui/layout/table/components/TableCell';
import { TableHeader } from '@/ui/layout/table/components/TableHeader';
import { TableRow } from '@/ui/layout/table/components/TableRow';
export const SettingsAdminHealthMessageSyncCountersTable = ({
details,
}: {
details: string | null | undefined;
}) => {
const parsedDetails = details ? JSON.parse(details) : null;
if (!parsedDetails) {
return null;
}
return (
<Table>
<TableRow>
<TableHeader>Status</TableHeader>
<TableHeader align="right">Count</TableHeader>
</TableRow>
<TableRow>
<TableCell>Message Not Synced</TableCell>
<TableCell align="right">{parsedDetails.counters.NOT_SYNCED}</TableCell>
</TableRow>
<TableRow>
<TableCell>Message Sync Ongoing</TableCell>
<TableCell align="right">{parsedDetails.counters.ONGOING}</TableCell>
</TableRow>
<TableRow>
<TableCell>Total Jobs</TableCell>
<TableCell align="right">{parsedDetails.totalJobs}</TableCell>
</TableRow>
<TableRow>
<TableCell>Failed Jobs</TableCell>
<TableCell align="right">{parsedDetails.failedJobs}</TableCell>
</TableRow>
<TableRow>
<TableCell>Failure Rate</TableCell>
<TableCell align="right">{parsedDetails.failureRate}%</TableCell>
</TableRow>
</Table>
);
};

66
packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatus.tsx
View File

@ -1,66 +0,0 @@
import { SettingsAdminHealthMessageSyncCountersTable } from '@/settings/admin-panel/components/SettingsAdminHealthMessageSyncCountersTable';
import { SettingsHealthStatusListCard } from '@/settings/admin-panel/components/SettingsHealthStatusListCard';
import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService';
import styled from '@emotion/styled';
import { H2Title, Section } from 'twenty-ui';
import {
AdminPanelHealthServiceStatus,
useGetSystemHealthStatusQuery,
} from '~/generated/graphql';
const StyledErrorMessage = styled.div`
color: ${({ theme }) => theme.color.red};
margin-top: ${({ theme }) => theme.spacing(2)};
`;
export const SettingsAdminHealthStatus = () => {
const { data, loading } = useGetSystemHealthStatusQuery({
fetchPolicy: 'network-only',
});
const services = [
{
id: 'DATABASE',
name: 'Database Status',
...data?.getSystemHealthStatus.database,
},
{ id: 'REDIS', name: 'Redis Status', ...data?.getSystemHealthStatus.redis },
{
id: 'WORKER',
name: 'Worker Status',
status: data?.getSystemHealthStatus.worker.status,
queues: data?.getSystemHealthStatus.worker.queues,
},
].filter((service): service is AdminHealthService => !!service.status);
const isMessageSyncCounterDown =
!data?.getSystemHealthStatus.messageSync.status ||
data?.getSystemHealthStatus.messageSync.status ===
AdminPanelHealthServiceStatus.OUTAGE;
return (
<>
<Section>
<H2Title title="Health Status" description="How your system is doing" />
<SettingsHealthStatusListCard services={services} loading={loading} />
</Section>
<Section>
<H2Title
title="Message Sync Status"
description="How your message sync is doing"
/>
{isMessageSyncCounterDown ? (
<StyledErrorMessage>
{data?.getSystemHealthStatus.messageSync.details ||
'Message sync status is unavailable'}
</StyledErrorMessage>
) : (
<SettingsAdminHealthMessageSyncCountersTable
details={data?.getSystemHealthStatus.messageSync.details}
/>
)}
</Section>
</>
);
};

27
packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminHealthStatusRightContainer.tsx
View File

@ -1,27 +0,0 @@
import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService';
import styled from '@emotion/styled';
import { Status } from 'twenty-ui';
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
const StyledRowRightContainer = styled.div`
align-items: center;
display: flex;
gap: ${({ theme }) => theme.spacing(1)};
`;
export const SettingsAdminHealthStatusRightContainer = ({
service,
}: {
service: AdminHealthService;
}) => {
return (
<StyledRowRightContainer>
{service.status === AdminPanelHealthServiceStatus.OPERATIONAL && (
<Status color="green" text="Operational" weight="medium" />
)}
{service.status === AdminPanelHealthServiceStatus.OUTAGE && (
<Status color="red" text="Outage" weight="medium" />
)}
</StyledRowRightContainer>
);
};

2
packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminTabContent.tsx
View File

@ -1,8 +1,8 @@
import { SettingsAdminEnvVariables } from '@/settings/admin-panel/components/SettingsAdminEnvVariables';
import { SettingsAdminGeneral } from '@/settings/admin-panel/components/SettingsAdminGeneral';
import { SettingsAdminHealthStatus } from '@/settings/admin-panel/components/SettingsAdminHealthStatus';
import { SETTINGS_ADMIN_TABS } from '@/settings/admin-panel/constants/SettingsAdminTabs';
import { SETTINGS_ADMIN_TABS_ID } from '@/settings/admin-panel/constants/SettingsAdminTabsId';
import { SettingsAdminHealthStatus } from '@/settings/admin-panel/health-status/components/SettingsAdminHealthStatus';
import { useTabList } from '@/ui/layout/tab/hooks/useTabList';
export const SettingsAdminTabContent = () => {

43
packages/twenty-front/src/modules/settings/admin-panel/components/SettingsHealthStatusListCard.tsx
View File

@ -1,43 +0,0 @@
import { AdminHealthService } from '@/settings/admin-panel/types/AdminHealthService';
import styled from '@emotion/styled';
import { SettingsPath } from '@/types/SettingsPath';
import { Link } from 'react-router-dom';
import { getSettingsPath } from '~/utils/navigation/getSettingsPath';
import { SettingsListCard } from '../../components/SettingsListCard';
import { SettingsAdminHealthStatusRightContainer } from './SettingsAdminHealthStatusRightContainer';
const StyledLink = styled(Link)`
text-decoration: none;
`;
export const SettingsHealthStatusListCard = ({
services,
loading,
}: {
services: Array<AdminHealthService>;
loading?: boolean;
}) => {
return (
<>
{services.map((service) => (
<>
<StyledLink
to={getSettingsPath(SettingsPath.AdminPanelIndicatorHealthStatus, {
indicatorName: service.id,
})}
>
<SettingsListCard
items={[service]}
getItemLabel={(service) => service.name}
isLoading={loading}
RowRightComponent={({ item: service }) => (
<SettingsAdminHealthStatusRightContainer service={service} />
)}
/>
</StyledLink>
</>
))}
</>
);
};

36
packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getSystemHealthStatus.ts
View File

@ -1,36 +0,0 @@
import { gql } from '@apollo/client';
export const GET_SYSTEM_HEALTH_STATUS = gql`
query GetSystemHealthStatus {
getSystemHealthStatus {
database {
status
details
}
redis {
status
details
}
worker {
status
queues {
name
workers
status
metrics {
failed
completed
waiting
active
delayed
prioritized
}
}
}
messageSync {
status
details
}
}
}
`;

57
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/ConnectedAccountHealthStatus.tsx Normal file
View File

@ -0,0 +1,57 @@
import { SettingsAdminHealthAccountSyncCountersTable } from '@/settings/admin-panel/health-status/components/SettingsAdminHealthAccountSyncCountersTable';
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
import styled from '@emotion/styled';
import { useContext } from 'react';
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
const StyledErrorMessage = styled.div`
color: ${({ theme }) => theme.color.red};
margin-top: ${({ theme }) => theme.spacing(2)};
`;
export const ConnectedAccountHealthStatus = () => {
const { indicatorHealth } = useContext(SettingsAdminIndicatorHealthContext);
const details = indicatorHealth.details;
if (!details) {
return null;
}
const parsedDetails = JSON.parse(details);
const isMessageSyncDown =
parsedDetails.messageSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
const isCalendarSyncDown =
parsedDetails.calendarSync?.status === AdminPanelHealthServiceStatus.OUTAGE;
const errorMessages = [];
if (isMessageSyncDown) {
errorMessages.push('Message Sync');
}
if (isCalendarSyncDown) {
errorMessages.push('Calendar Sync');
}
return (
<>
{errorMessages.length > 0 && (
<StyledErrorMessage>
{`${errorMessages.join(' and ')} ${errorMessages.length > 1 ? 'are' : 'is'} not available because the service is down`}
</StyledErrorMessage>
)}
{!isMessageSyncDown && parsedDetails.messageSync?.details && (
<SettingsAdminHealthAccountSyncCountersTable
details={parsedDetails.messageSync.details}
title="Message Sync Status"
/>
)}
{!isCalendarSyncDown && parsedDetails.calendarSync?.details && (
<SettingsAdminHealthAccountSyncCountersTable
details={parsedDetails.calendarSync.details}
title="Calendar Sync Status"
/>
)}
</>
);
};

45
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/DatabaseAndRedisHealthStatus.tsx Normal file
View File

@ -0,0 +1,45 @@
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
import styled from '@emotion/styled';
import { useContext } from 'react';
import { Section } from 'twenty-ui';
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
const StyledDetailsContainer = styled.pre`
background-color: ${({ theme }) => theme.background.quaternary};
padding: ${({ theme }) => theme.spacing(6)};
border-radius: ${({ theme }) => theme.border.radius.sm};
white-space: pre-wrap;
font-size: ${({ theme }) => theme.font.size.sm};
margin: 0;
`;
const StyledErrorMessage = styled.div`
color: ${({ theme }) => theme.color.red};
margin-top: ${({ theme }) => theme.spacing(2)};
`;
export const DatabaseAndRedisHealthStatus = () => {
const { indicatorHealth, loading } = useContext(
SettingsAdminIndicatorHealthContext,
);
const formattedDetails = indicatorHealth.details
? JSON.stringify(JSON.parse(indicatorHealth.details), null, 2)
: null;
const isDatabaseOrRedisDown =
!indicatorHealth.status ||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
return (
<Section>
{isDatabaseOrRedisDown && !loading ? (
<StyledErrorMessage>
{`${indicatorHealth.label} information is not available because the service is down`}
</StyledErrorMessage>
) : (
<StyledDetailsContainer>{formattedDetails}</StyledDetailsContainer>
)}
</Section>
);
};

55
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminHealthAccountSyncCountersTable.tsx Normal file
View File

@ -0,0 +1,55 @@
import { Table } from '@/ui/layout/table/components/Table';
import { TableCell } from '@/ui/layout/table/components/TableCell';
import { TableHeader } from '@/ui/layout/table/components/TableHeader';
import { TableRow } from '@/ui/layout/table/components/TableRow';
import styled from '@emotion/styled';
import { H2Title } from 'twenty-ui';
const StyledContainer = styled.div``;
export const SettingsAdminHealthAccountSyncCountersTable = ({
details,
title,
}: {
details: Record<string, any> | null;
title: string;
}) => {
if (!details) {
return null;
}
return (
<StyledContainer>
<H2Title
title={title}
description={`How your ${title.toLowerCase()} is doing`}
/>
<Table>
<TableRow>
<TableHeader>Status</TableHeader>
<TableHeader align="right">Count</TableHeader>
</TableRow>
<TableRow>
<TableCell>Not Synced</TableCell>
<TableCell align="right">{details.counters.NOT_SYNCED}</TableCell>
</TableRow>
<TableRow>
<TableCell>Sync Ongoing</TableCell>
<TableCell align="right">{details.counters.ONGOING}</TableCell>
</TableRow>
<TableRow>
<TableCell>Total Jobs</TableCell>
<TableCell align="right">{details.totalJobs}</TableCell>
</TableRow>
<TableRow>
<TableCell>Failed Jobs</TableCell>
<TableCell align="right">{details.failedJobs}</TableCell>
</TableRow>
<TableRow>
<TableCell>Failure Rate</TableCell>
<TableCell align="right">{details.failureRate}%</TableCell>
</TableRow>
</Table>
</StyledContainer>
);
};

19
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminHealthStatus.tsx Normal file
View File

@ -0,0 +1,19 @@
import { SettingsHealthStatusListCard } from '@/settings/admin-panel/health-status/components/SettingsHealthStatusListCard';
import { H2Title, Section } from 'twenty-ui';
import { useGetSystemHealthStatusQuery } from '~/generated/graphql';
export const SettingsAdminHealthStatus = () => {
const { data, loading } = useGetSystemHealthStatusQuery({
fetchPolicy: 'network-only',
});
const services = data?.getSystemHealthStatus.services ?? [];
return (
<>
<Section>
<H2Title title="Health Status" description="How your system is doing" />
<SettingsHealthStatusListCard services={services} loading={loading} />
</Section>
</>
);
};

19
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminHealthStatusRightContainer.tsx Normal file
View File

@ -0,0 +1,19 @@
import { Status } from 'twenty-ui';
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
export const SettingsAdminHealthStatusRightContainer = ({
status,
}: {
status: AdminPanelHealthServiceStatus;
}) => {
return (
<>
{status === AdminPanelHealthServiceStatus.OPERATIONAL && (
<Status color="green" text="Operational" weight="medium" />
)}
{status === AdminPanelHealthServiceStatus.OUTAGE && (
<Status color="red" text="Outage" weight="medium" />
)}
</>
);
};

21
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminIndicatorHealthStatusContent.tsx Normal file
View File

@ -0,0 +1,21 @@
import { ConnectedAccountHealthStatus } from '@/settings/admin-panel/health-status/components/ConnectedAccountHealthStatus';
import { DatabaseAndRedisHealthStatus } from '@/settings/admin-panel/health-status/components/DatabaseAndRedisHealthStatus';
import { WorkerHealthStatus } from '@/settings/admin-panel/health-status/components/WorkerHealthStatus';
import { useParams } from 'react-router-dom';
import { HealthIndicatorId } from '~/generated/graphql';
export const SettingsAdminIndicatorHealthStatusContent = () => {
const { indicatorId } = useParams();
switch (indicatorId) {
case HealthIndicatorId.database:
case HealthIndicatorId.redis:
return <DatabaseAndRedisHealthStatus />;
case HealthIndicatorId.worker:
return <WorkerHealthStatus />;
case HealthIndicatorId.connectedAccount:
return <ConnectedAccountHealthStatus />;
default:
return null;
}
};

8
packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueExpandableContainer.tsx → packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminQueueExpandableContainer.tsx
View File

@ -43,7 +43,7 @@ export const SettingsAdminQueueExpandableContainer = ({
selectedQueue: string | null;
}) => {
const selectedQueueData = queues.find(
(queue) => queue.name === selectedQueue,
(queue) => queue.queueName === selectedQueue,
);
return (
@ -55,10 +55,12 @@ export const SettingsAdminQueueExpandableContainer = ({
<>
<StyledContainer>
<SettingsListCard
items={[{ ...selectedQueueData, id: selectedQueueData.name }]}
items={[
{ ...selectedQueueData, id: selectedQueueData.queueName },
]}
getItemLabel={(
item: AdminPanelWorkerQueueHealth & { id: string },
) => item.name}
) => item.queueName}
isLoading={false}
RowRightComponent={({
item,

8
packages/twenty-front/src/modules/settings/admin-panel/components/SettingsAdminQueueHealthButtons.tsx → packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsAdminQueueHealthButtons.tsx
View File

@ -39,11 +39,11 @@ export const SettingsAdminQueueHealthButtons = ({
<StyledQueueButtonsRow>
{queues.map((queue) => (
<StyledQueueHealthButton
key={queue.name}
onClick={() => toggleQueueVisibility(queue.name)}
title={queue.name}
key={queue.queueName}
onClick={() => toggleQueueVisibility(queue.queueName)}
title={queue.queueName}
variant="secondary"
isSelected={selectedQueue === queue.name}
isSelected={selectedQueue === queue.queueName}
status={queue.status}
/>
))}

29
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/SettingsHealthStatusListCard.tsx Normal file
View File

@ -0,0 +1,29 @@
import { SettingsListCard } from '@/settings/components/SettingsListCard';
import { SettingsPath } from '@/types/SettingsPath';
import { SystemHealthService } from '~/generated/graphql';
import { getSettingsPath } from '~/utils/navigation/getSettingsPath';
import { SettingsAdminHealthStatusRightContainer } from './SettingsAdminHealthStatusRightContainer';
export const SettingsHealthStatusListCard = ({
services,
loading,
}: {
services: Array<SystemHealthService>;
loading?: boolean;
}) => {
return (
<SettingsListCard
items={services}
getItemLabel={(service) => service.label}
isLoading={loading}
RowRightComponent={({ item: service }) => (
<SettingsAdminHealthStatusRightContainer status={service.status} />
)}
to={(service) =>
getSettingsPath(SettingsPath.AdminPanelIndicatorHealthStatus, {
indicatorId: service.id,
})
}
/>
);
};

62
packages/twenty-front/src/modules/settings/admin-panel/health-status/components/WorkerHealthStatus.tsx Normal file
View File

@ -0,0 +1,62 @@
import { SettingsAdminQueueExpandableContainer } from '@/settings/admin-panel/health-status/components/SettingsAdminQueueExpandableContainer';
import { SettingsAdminQueueHealthButtons } from '@/settings/admin-panel/health-status/components/SettingsAdminQueueHealthButtons';
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
import styled from '@emotion/styled';
import { useContext, useState } from 'react';
import { H2Title, Section } from 'twenty-ui';
import { AdminPanelHealthServiceStatus } from '~/generated/graphql';
const StyledTitleContainer = styled.div`
display: flex;
flex-direction: column;
flex: 1;
`;
const StyledErrorMessage = styled.div`
color: ${({ theme }) => theme.color.red};
margin-top: ${({ theme }) => theme.spacing(2)};
`;
export const WorkerHealthStatus = () => {
const { indicatorHealth, loading } = useContext(
SettingsAdminIndicatorHealthContext,
);
const isWorkerDown =
!indicatorHealth.status ||
indicatorHealth.status === AdminPanelHealthServiceStatus.OUTAGE;
const [selectedQueue, setSelectedQueue] = useState<string | null>(null);
const toggleQueueVisibility = (queueName: string) => {
setSelectedQueue(selectedQueue === queueName ? null : queueName);
};
return (
<Section>
<StyledTitleContainer>
<H2Title
title="Queue Status"
description="Background job processing status and metrics"
/>
</StyledTitleContainer>
{isWorkerDown && !loading ? (
<StyledErrorMessage>
Queue information is not available because the worker is down
</StyledErrorMessage>
) : (
<>
<SettingsAdminQueueHealthButtons
queues={indicatorHealth.queues ?? []}
selectedQueue={selectedQueue}
toggleQueueVisibility={toggleQueueVisibility}
/>
<SettingsAdminQueueExpandableContainer
queues={indicatorHealth.queues ?? []}
selectedQueue={selectedQueue}
/>
</>
)}
</Section>
);
};

23
packages/twenty-front/src/modules/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext.tsx Normal file
View File

@ -0,0 +1,23 @@
import { createContext } from 'react';
import {
AdminPanelHealthServiceData,
AdminPanelHealthServiceStatus,
} from '~/generated/graphql';
type SettingsAdminIndicatorHealthContextType = {
indicatorHealth: AdminPanelHealthServiceData;
loading: boolean;
};
export const SettingsAdminIndicatorHealthContext =
createContext<SettingsAdminIndicatorHealthContextType>({
indicatorHealth: {
id: '',
label: '',
description: '',
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: '',
queues: [],
},
loading: false,
});

12
packages/twenty-front/src/modules/settings/admin-panel/graphql/queries/getIndicatorHealthStatus.ts → packages/twenty-front/src/modules/settings/admin-panel/health-status/graphql/queries/getIndicatorHealthStatus.ts
View File

@ -1,14 +1,16 @@
import { gql } from '@apollo/client';
export const GET_INDICATOR_HEALTH_STATUS = gql`
query GetIndicatorHealthStatus(
$indicatorName: AdminPanelIndicatorHealthStatusInputEnum!
) {
getIndicatorHealthStatus(indicatorName: $indicatorName) {
query GetIndicatorHealthStatus($indicatorId: HealthIndicatorId!) {
getIndicatorHealthStatus(indicatorId: $indicatorId) {
id
label
description
status
details
queues {
name
id
queueName
status
workers
metrics {

13
packages/twenty-front/src/modules/settings/admin-panel/health-status/graphql/queries/getSystemHealthStatus.ts Normal file
View File

@ -0,0 +1,13 @@
import { gql } from '@apollo/client';
export const GET_SYSTEM_HEALTH_STATUS = gql`
query GetSystemHealthStatus {
getSystemHealthStatus {
services {
id
label
status
}
}
}
`;

12
packages/twenty-front/src/modules/settings/admin-panel/hooks/useGetUptoDateHealthStatus.ts
View File

@ -1,12 +0,0 @@
import { useGetSystemHealthStatusQuery } from '~/generated/graphql';
export const useGetUptoDateHealthStatus = () => {
const { data, loading } = useGetSystemHealthStatusQuery({
fetchPolicy: 'network-only',
});
return {
healthStatus: data?.getSystemHealthStatus,
healthStatusLoading: loading,
};
};

12
packages/twenty-front/src/modules/settings/admin-panel/types/AdminHealthService.ts
View File

@ -1,12 +0,0 @@
import {
AdminPanelHealthServiceData,
AdminPanelWorkerQueueHealth,
} from '~/generated/graphql';
type AdminWorkerService = AdminPanelHealthServiceData & {
id: string;
name: string;
queues: AdminPanelWorkerQueueHealth[] | null | undefined;
};
export type AdminHealthService = AdminWorkerService;

2
packages/twenty-front/src/modules/settings/components/SettingsCard.tsx
View File

@ -1,6 +1,6 @@
import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { IconChevronRight, Pill, Card, CardContent } from 'twenty-ui';
import { Card, CardContent, IconChevronRight, Pill } from 'twenty-ui';
import { ReactNode } from 'react';

5
packages/twenty-front/src/modules/settings/components/SettingsListCard.tsx
View File

@ -1,7 +1,7 @@
import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { ComponentType } from 'react';
import { IconComponent, IconPlus, Card, CardFooter } from 'twenty-ui';
import { Card, CardFooter, IconComponent, IconPlus } from 'twenty-ui';
import { SettingsListSkeletonCard } from '@/settings/components/SettingsListSkeletonCard';
@ -44,6 +44,7 @@ type SettingsListCardProps<ListItem extends { id: string }> = {
RowRightComponent: ComponentType<{ item: ListItem }>;
footerButtonLabel?: string;
onFooterButtonClick?: () => void;
to?: (item: ListItem) => string;
};
export const SettingsListCard = <
@ -61,6 +62,7 @@ export const SettingsListCard = <
RowRightComponent,
onFooterButtonClick,
footerButtonLabel,
to,
}: SettingsListCardProps<ListItem>) => {
const theme = useTheme();
@ -76,6 +78,7 @@ export const SettingsListCard = <
rightComponent={<RowRightComponent item={item} />}
divider={index < items.length - 1}
onClick={() => onRowClick?.(item)}
to={to?.(item)}
/>
))}
{hasFooter && (

23
packages/twenty-front/src/modules/settings/components/SettingsListItemCardContent.tsx
View File

@ -1,7 +1,9 @@
import { useTheme } from '@emotion/react';
import styled from '@emotion/styled';
import { ReactNode } from 'react';
import { IconComponent, CardContent } from 'twenty-ui';
import { Link } from 'react-router-dom';
import { isDefined } from 'twenty-shared';
import { CardContent, IconComponent } from 'twenty-ui';
const StyledRow = styled(CardContent)`
align-items: center;
@ -19,12 +21,22 @@ const StyledLabel = styled.span`
flex: 1 0 auto;
`;
const StyledLink = styled(Link)`
text-decoration: none;
color: ${({ theme }) => theme.font.color.secondary};
&:hover {
color: ${({ theme }) => theme.font.color.secondary};
}
`;
type SettingsListItemCardContentProps = {
label: string;
divider?: boolean;
LeftIcon?: IconComponent;
onClick?: () => void;
rightComponent: ReactNode;
to?: string;
};
export const SettingsListItemCardContent = ({
@ -33,14 +45,21 @@ export const SettingsListItemCardContent = ({
LeftIcon,
onClick,
rightComponent,
to,
}: SettingsListItemCardContentProps) => {
const theme = useTheme();
return (
const content = (
<StyledRow onClick={onClick} divider={divider}>
{!!LeftIcon && <LeftIcon size={theme.icon.size.md} />}
<StyledLabel>{label}</StyledLabel>
{rightComponent}
</StyledRow>
);
if (isDefined(to)) {
return <StyledLink to={to}>{content}</StyledLink>;
}
return content;
};

2
packages/twenty-front/src/modules/types/SettingsPath.ts
View File

@ -35,7 +35,7 @@ export enum SettingsPath {
AdminPanel = 'admin-panel',
FeatureFlags = 'admin-panel/feature-flags',
AdminPanelHealthStatus = 'admin-panel#health-status',
AdminPanelIndicatorHealthStatus = 'admin-panel/health-status/:indicatorName',
AdminPanelIndicatorHealthStatus = 'admin-panel/health-status/:indicatorId',
Lab = 'lab',
Roles = 'roles',
RoleDetail = 'roles/:roleId',

125
packages/twenty-front/src/pages/settings/admin-panel/SettingsAdminIndicatorHealthStatus.tsx
View File

@ -1,65 +1,34 @@
import { SettingsAdminQueueExpandableContainer } from '@/settings/admin-panel/components/SettingsAdminQueueExpandableContainer';
import { SettingsAdminQueueHealthButtons } from '@/settings/admin-panel/components/SettingsAdminQueueHealthButtons';
import { SettingsAdminHealthStatusRightContainer } from '@/settings/admin-panel/health-status/components/SettingsAdminHealthStatusRightContainer';
import { SettingsAdminIndicatorHealthStatusContent } from '@/settings/admin-panel/health-status/components/SettingsAdminIndicatorHealthStatusContent';
import { SettingsAdminIndicatorHealthContext } from '@/settings/admin-panel/health-status/contexts/SettingsAdminIndicatorHealthContext';
import { SettingsPageContainer } from '@/settings/components/SettingsPageContainer';
import { SettingsPath } from '@/types/SettingsPath';
import { SubMenuTopBarContainer } from '@/ui/layout/page/components/SubMenuTopBarContainer';
import styled from '@emotion/styled';
import { useLingui } from '@lingui/react/macro';
import { useState } from 'react';
import { useParams } from 'react-router-dom';
import { H2Title, Section, Status } from 'twenty-ui';
import { H2Title, Section } from 'twenty-ui';
import {
AdminPanelHealthServiceStatus,
AdminPanelIndicatorHealthStatusInputEnum,
HealthIndicatorId,
useGetIndicatorHealthStatusQuery,
} from '~/generated/graphql';
import { getSettingsPath } from '~/utils/navigation/getSettingsPath';
const StyledStatusContainer = styled.div``;
const StyledTitleContainer = styled.div`
display: flex;
flex-direction: column;
flex: 1;
`;
const StyledErrorMessage = styled.div`
color: ${({ theme }) => theme.color.red};
const StyledH2Title = styled(H2Title)`
margin-top: ${({ theme }) => theme.spacing(2)};
`;
const StyledDetailsContainer = styled.pre`
background-color: ${({ theme }) => theme.background.quaternary};
padding: ${({ theme }) => theme.spacing(6)};
border-radius: ${({ theme }) => theme.border.radius.sm};
white-space: pre-wrap;
font-size: ${({ theme }) => theme.font.size.sm};
`;
export const SettingsAdminIndicatorHealthStatus = () => {
const { t } = useLingui();
const { indicatorName } = useParams();
const { indicatorId } = useParams();
const { data, loading } = useGetIndicatorHealthStatusQuery({
variables: {
indicatorName: indicatorName as AdminPanelIndicatorHealthStatusInputEnum,
indicatorId: indicatorId as HealthIndicatorId,
},
fetchPolicy: 'network-only',
});
const formattedDetails = data?.getIndicatorHealthStatus.details
? JSON.stringify(JSON.parse(data.getIndicatorHealthStatus.details), null, 2)
: null;
const isWorkerDown =
!data?.getIndicatorHealthStatus.status ||
data?.getIndicatorHealthStatus.status ===
AdminPanelHealthServiceStatus.OUTAGE;
const [selectedQueue, setSelectedQueue] = useState<string | null>(null);
const toggleQueueVisibility = (queueName: string) => {
setSelectedQueue(selectedQueue === queueName ? null : queueName);
};
return (
<SubMenuTopBarContainer
links={[
@ -75,62 +44,40 @@ export const SettingsAdminIndicatorHealthStatus = () => {
children: t`Health Status`,
href: getSettingsPath(SettingsPath.AdminPanelHealthStatus),
},
{ children: `${indicatorName}` },
{ children: `${data?.getIndicatorHealthStatus?.label}` },
]}
>
<SettingsPageContainer>
<Section>
<H2Title title={`${indicatorName}`} description="Health status" />
<StyledStatusContainer>
{data?.getIndicatorHealthStatus.status ===
AdminPanelHealthServiceStatus.OPERATIONAL && (
<Status color="green" text="Operational" weight="medium" />
)}
{data?.getIndicatorHealthStatus.status ===
AdminPanelHealthServiceStatus.OUTAGE && (
<Status color="red" text="Outage" weight="medium" />
)}
</StyledStatusContainer>
</Section>
{indicatorName === AdminPanelIndicatorHealthStatusInputEnum.WORKER ? (
<SettingsAdminIndicatorHealthContext.Provider
value={{
indicatorHealth: {
id: data?.getIndicatorHealthStatus?.id ?? '',
label: data?.getIndicatorHealthStatus?.label ?? '',
description: data?.getIndicatorHealthStatus?.description ?? '',
status:
data?.getIndicatorHealthStatus?.status ??
AdminPanelHealthServiceStatus.OUTAGE,
details: data?.getIndicatorHealthStatus?.details,
queues: data?.getIndicatorHealthStatus?.queues,
},
loading: loading,
}}
>
<Section>
<StyledTitleContainer>
<H2Title
title="Queue Status"
description="Background job processing status and metrics"
/>
</StyledTitleContainer>
{isWorkerDown && !loading ? (
<StyledErrorMessage>
Queue information is not available because the worker is down
</StyledErrorMessage>
) : (
<>
<SettingsAdminQueueHealthButtons
queues={data?.getIndicatorHealthStatus.queues ?? []}
selectedQueue={selectedQueue}
toggleQueueVisibility={toggleQueueVisibility}
<StyledH2Title
title={`${data?.getIndicatorHealthStatus?.label}`}
description={data?.getIndicatorHealthStatus?.description}
/>
{indicatorId !== HealthIndicatorId.connectedAccount &&
data?.getIndicatorHealthStatus?.status && (
<SettingsAdminHealthStatusRightContainer
status={data?.getIndicatorHealthStatus.status}
/>
<SettingsAdminQueueExpandableContainer
queues={data?.getIndicatorHealthStatus.queues ?? []}
selectedQueue={selectedQueue}
/>
</>
)}
)}
</Section>
) : null}
{indicatorName === AdminPanelIndicatorHealthStatusInputEnum.DATABASE ||
indicatorName === AdminPanelIndicatorHealthStatusInputEnum.REDIS ? (
<Section>
{formattedDetails && (
<StyledDetailsContainer>
{formattedDetails}
</StyledDetailsContainer>
)}
</Section>
) : null}
<SettingsAdminIndicatorHealthStatusContent />
</SettingsAdminIndicatorHealthContext.Provider>
</SettingsPageContainer>
</SubMenuTopBarContainer>
);

379
packages/twenty-server/src/engine/core-modules/admin-panel/__tests__/admin-panel-health.service.spec.ts
View File

@ -1,11 +1,13 @@
import { Test, TestingModule } from '@nestjs/testing';
import { AdminPanelHealthService } from 'src/engine/core-modules/admin-panel/admin-panel-health.service';
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
@ -14,44 +16,21 @@ describe('AdminPanelHealthService', () => {
let databaseHealth: jest.Mocked<DatabaseHealthIndicator>;
let redisHealth: jest.Mocked<RedisHealthIndicator>;
let workerHealth: jest.Mocked<WorkerHealthIndicator>;
let messageSyncHealth: jest.Mocked<MessageSyncHealthIndicator>;
let connectedAccountHealth: jest.Mocked<ConnectedAccountHealth>;
beforeEach(async () => {
databaseHealth = {
isHealthy: jest.fn(),
} as any;
redisHealth = {
isHealthy: jest.fn(),
} as any;
workerHealth = {
isHealthy: jest.fn(),
} as any;
messageSyncHealth = {
isHealthy: jest.fn(),
} as any;
databaseHealth = { isHealthy: jest.fn() } as any;
redisHealth = { isHealthy: jest.fn() } as any;
workerHealth = { isHealthy: jest.fn() } as any;
connectedAccountHealth = { isHealthy: jest.fn() } as any;
const module: TestingModule = await Test.createTestingModule({
providers: [
AdminPanelHealthService,
{
provide: DatabaseHealthIndicator,
useValue: databaseHealth,
},
{
provide: RedisHealthIndicator,
useValue: redisHealth,
},
{
provide: WorkerHealthIndicator,
useValue: workerHealth,
},
{
provide: MessageSyncHealthIndicator,
useValue: messageSyncHealth,
},
{ provide: DatabaseHealthIndicator, useValue: databaseHealth },
{ provide: RedisHealthIndicator, useValue: redisHealth },
{ provide: WorkerHealthIndicator, useValue: workerHealth },
{ provide: ConnectedAccountHealth, useValue: connectedAccountHealth },
],
}).compile();
@ -62,132 +41,244 @@ describe('AdminPanelHealthService', () => {
expect(service).toBeDefined();
});
it('should transform health check response to SystemHealth format', async () => {
databaseHealth.isHealthy.mockResolvedValue({
database: {
status: 'up',
details: 'Database is healthy',
},
});
redisHealth.isHealthy.mockResolvedValue({
redis: {
status: 'up',
details: 'Redis is connected',
},
});
workerHealth.isHealthy.mockResolvedValue({
worker: {
status: 'up',
queues: [
{
name: 'test',
workers: 1,
metrics: {
active: 1,
completed: 0,
delayed: 4,
failed: 3,
waiting: 0,
prioritized: 0,
describe('getSystemHealthStatus', () => {
it('should transform health check response to SystemHealth format', async () => {
databaseHealth.isHealthy.mockResolvedValue({
database: { status: 'up', details: 'Database is healthy' },
});
redisHealth.isHealthy.mockResolvedValue({
redis: { status: 'up', details: 'Redis is connected' },
});
workerHealth.isHealthy.mockResolvedValue({
worker: {
status: 'up',
queues: [
{
queueName: 'test',
workers: 1,
metrics: {
active: 1,
completed: 0,
delayed: 4,
failed: 3,
waiting: 0,
prioritized: 0,
},
},
],
},
});
connectedAccountHealth.isHealthy.mockResolvedValue({
connectedAccount: {
status: 'up',
details: 'Account sync is operational',
},
});
const result = await service.getSystemHealthStatus();
const expected: SystemHealth = {
services: [
{
...HEALTH_INDICATORS[HealthIndicatorId.database],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.redis],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.worker],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
],
},
});
messageSyncHealth.isHealthy.mockResolvedValue({
messageSync: {
status: 'up',
details: 'Message sync is operational',
},
};
expect(result).toStrictEqual(expected);
});
const result = await service.getSystemHealthStatus();
it('should handle mixed health statuses', async () => {
databaseHealth.isHealthy.mockResolvedValue({
database: { status: 'up' },
});
redisHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
workerHealth.isHealthy.mockResolvedValue({
worker: { status: 'up', queues: [] },
});
connectedAccountHealth.isHealthy.mockResolvedValue({
connectedAccount: { status: 'up' },
});
const expected: SystemHealth = {
database: {
const result = await service.getSystemHealthStatus();
expect(result).toStrictEqual({
services: [
{
...HEALTH_INDICATORS[HealthIndicatorId.database],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.redis],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.worker],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
},
],
});
});
it('should handle all services down', async () => {
databaseHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
);
redisHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
workerHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS),
);
connectedAccountHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED),
);
const result = await service.getSystemHealthStatus();
expect(result).toStrictEqual({
services: [
{
...HEALTH_INDICATORS[HealthIndicatorId.database],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.redis],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.worker],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: AdminPanelHealthServiceStatus.OUTAGE,
},
],
});
});
});
describe('getIndicatorHealthStatus', () => {
it('should return health status for database indicator', async () => {
const details = {
version: '15.0',
connections: { active: 5, max: 100 },
};
databaseHealth.isHealthy.mockResolvedValue({
database: {
status: 'up',
details,
},
});
const result = await service.getIndicatorHealthStatus(
HealthIndicatorId.database,
);
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.database],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: '"Database is healthy"',
details: JSON.stringify(details),
queues: undefined,
},
redis: {
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: '"Redis is connected"',
queues: undefined,
},
worker: {
});
});
it('should return health status with queues for worker indicator', async () => {
const mockQueues = [
{
queueName: 'queue1',
workers: 2,
metrics: {
active: 1,
completed: 10,
delayed: 0,
failed: 2,
waiting: 5,
prioritized: 1,
},
},
{
queueName: 'queue2',
workers: 0,
metrics: {
active: 0,
completed: 5,
delayed: 0,
failed: 1,
waiting: 2,
prioritized: 0,
},
},
];
workerHealth.isHealthy.mockResolvedValue({
worker: {
status: 'up',
queues: mockQueues,
},
});
const result = await service.getIndicatorHealthStatus(
HealthIndicatorId.worker,
);
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.worker],
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: undefined,
queues: [
{
name: 'test',
workers: 1,
status: AdminPanelHealthServiceStatus.OPERATIONAL,
metrics: {
active: 1,
completed: 0,
delayed: 4,
failed: 3,
waiting: 0,
prioritized: 0,
},
},
],
},
messageSync: {
status: AdminPanelHealthServiceStatus.OPERATIONAL,
details: '"Message sync is operational"',
queues: undefined,
},
};
expect(result).toStrictEqual(expected);
});
it('should handle mixed health statuses', async () => {
databaseHealth.isHealthy.mockResolvedValue({
database: { status: 'up' },
});
redisHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
workerHealth.isHealthy.mockResolvedValue({
worker: { status: 'up', queues: [] },
});
messageSyncHealth.isHealthy.mockResolvedValue({
messageSync: { status: 'up' },
queues: mockQueues.map((queue) => ({
...queue,
id: `worker-${queue.queueName}`,
status:
queue.workers > 0
? AdminPanelHealthServiceStatus.OPERATIONAL
: AdminPanelHealthServiceStatus.OUTAGE,
})),
});
});
const result = await service.getSystemHealthStatus();
it('should handle failed indicator health check', async () => {
redisHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
expect(result).toMatchObject({
database: { status: AdminPanelHealthServiceStatus.OPERATIONAL },
redis: { status: AdminPanelHealthServiceStatus.OUTAGE },
worker: { status: AdminPanelHealthServiceStatus.OPERATIONAL },
messageSync: { status: AdminPanelHealthServiceStatus.OPERATIONAL },
const result = await service.getIndicatorHealthStatus(
HealthIndicatorId.redis,
);
expect(result).toStrictEqual({
...HEALTH_INDICATORS[HealthIndicatorId.redis],
status: AdminPanelHealthServiceStatus.OUTAGE,
details: HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED,
});
});
});
it('should handle all services down', async () => {
databaseHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.DATABASE_CONNECTION_FAILED),
);
redisHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.REDIS_CONNECTION_FAILED),
);
workerHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.NO_ACTIVE_WORKERS),
);
messageSyncHealth.isHealthy.mockRejectedValue(
new Error(HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED),
);
const result = await service.getSystemHealthStatus();
expect(result).toMatchObject({
database: { status: AdminPanelHealthServiceStatus.OUTAGE },
redis: { status: AdminPanelHealthServiceStatus.OUTAGE },
worker: { status: AdminPanelHealthServiceStatus.OUTAGE },
messageSync: { status: AdminPanelHealthServiceStatus.OUTAGE },
it('should throw error for invalid indicator', async () => {
await expect(
// @ts-expect-error Testing invalid input
service.getIndicatorHealthStatus('invalid'),
).rejects.toThrow('Health indicator not found: invalid');
});
});
});

111
packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel-health.service.ts
View File

@ -1,12 +1,13 @@
import { Injectable } from '@nestjs/common';
import { HealthIndicatorResult } from '@nestjs/terminus';
import { HealthIndicatorResult, HealthIndicatorStatus } from '@nestjs/terminus';
import { HEALTH_INDICATORS } from 'src/engine/core-modules/admin-panel/constants/health-indicators.constants';
import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto';
import { AdminPanelIndicatorHealthStatusInputEnum } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input';
import { SystemHealth } from 'src/engine/core-modules/admin-panel/dtos/system-health.dto';
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
@ -16,57 +17,84 @@ export class AdminPanelHealthService {
private readonly databaseHealth: DatabaseHealthIndicator,
private readonly redisHealth: RedisHealthIndicator,
private readonly workerHealth: WorkerHealthIndicator,
private readonly messageSyncHealth: MessageSyncHealthIndicator,
private readonly connectedAccountHealth: ConnectedAccountHealth,
) {}
private readonly healthIndicators = {
database: this.databaseHealth,
redis: this.redisHealth,
worker: this.workerHealth,
messageSync: this.messageSyncHealth,
[HealthIndicatorId.database]: this.databaseHealth,
[HealthIndicatorId.redis]: this.redisHealth,
[HealthIndicatorId.worker]: this.workerHealth,
[HealthIndicatorId.connectedAccount]: this.connectedAccountHealth,
};
private transformStatus(status: HealthIndicatorStatus) {
return status === 'up'
? AdminPanelHealthServiceStatus.OPERATIONAL
: AdminPanelHealthServiceStatus.OUTAGE;
}
private transformServiceDetails(details: any) {
if (!details) return details;
if (details.messageSync) {
details.messageSync.status = this.transformStatus(
details.messageSync.status,
);
}
if (details.calendarSync) {
details.calendarSync.status = this.transformStatus(
details.calendarSync.status,
);
}
return details;
}
private getServiceStatus(
result: PromiseSettledResult<HealthIndicatorResult>,
indicatorId: HealthIndicatorId,
) {
if (result.status === 'fulfilled') {
const key = Object.keys(result.value)[0];
const serviceResult = result.value[key];
const details = serviceResult.details;
const details = this.transformServiceDetails(serviceResult.details);
const indicator = HEALTH_INDICATORS[indicatorId];
return {
status:
serviceResult.status === 'up'
? AdminPanelHealthServiceStatus.OPERATIONAL
: AdminPanelHealthServiceStatus.OUTAGE,
id: indicatorId,
label: indicator.label,
description: indicator.description,
status: this.transformStatus(serviceResult.status),
details: details ? JSON.stringify(details) : undefined,
queues: serviceResult.queues,
};
}
return {
...HEALTH_INDICATORS[indicatorId],
status: AdminPanelHealthServiceStatus.OUTAGE,
details: result.reason?.message,
details: result.reason?.message?.toString(),
};
}
async getIndicatorHealthStatus(
indicatorName: AdminPanelIndicatorHealthStatusInputEnum,
indicatorId: HealthIndicatorId,
): Promise<AdminPanelHealthServiceData> {
const healthIndicator = this.healthIndicators[indicatorName];
const healthIndicator = this.healthIndicators[indicatorId];
if (!healthIndicator) {
throw new Error(`Health indicator not found: ${indicatorName}`);
throw new Error(`Health indicator not found: ${indicatorId}`);
}
const result = await Promise.allSettled([healthIndicator.isHealthy()]);
const indicatorStatus = this.getServiceStatus(result[0]);
const indicatorStatus = this.getServiceStatus(result[0], indicatorId);
if (indicatorName === 'worker') {
if (indicatorId === HealthIndicatorId.worker) {
return {
...indicatorStatus,
queues: (indicatorStatus?.queues ?? []).map((queue) => ({
...queue,
id: `${indicatorId}-${queue.queueName}`,
status:
queue.workers > 0
? AdminPanelHealthServiceStatus.OPERATIONAL
@ -79,30 +107,41 @@ export class AdminPanelHealthService {
}
async getSystemHealthStatus(): Promise<SystemHealth> {
const [databaseResult, redisResult, workerResult, messageSyncResult] =
const [databaseResult, redisResult, workerResult, accountSyncResult] =
await Promise.allSettled([
this.databaseHealth.isHealthy(),
this.redisHealth.isHealthy(),
this.workerHealth.isHealthy(),
this.messageSyncHealth.isHealthy(),
this.connectedAccountHealth.isHealthy(),
]);
const workerStatus = this.getServiceStatus(workerResult);
return {
database: this.getServiceStatus(databaseResult),
redis: this.getServiceStatus(redisResult),
worker: {
...workerStatus,
queues: (workerStatus?.queues ?? []).map((queue) => ({
...queue,
status:
queue.workers > 0
? AdminPanelHealthServiceStatus.OPERATIONAL
: AdminPanelHealthServiceStatus.OUTAGE,
})),
},
messageSync: this.getServiceStatus(messageSyncResult),
services: [
{
...HEALTH_INDICATORS[HealthIndicatorId.database],
status: this.getServiceStatus(
databaseResult,
HealthIndicatorId.database,
).status,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.redis],
status: this.getServiceStatus(redisResult, HealthIndicatorId.redis)
.status,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.worker],
status: this.getServiceStatus(workerResult, HealthIndicatorId.worker)
.status,
},
{
...HEALTH_INDICATORS[HealthIndicatorId.connectedAccount],
status: this.getServiceStatus(
accountSyncResult,
HealthIndicatorId.connectedAccount,
).status,
},
],
};
}
}

10
packages/twenty-server/src/engine/core-modules/admin-panel/admin-panel.resolver.ts
View File

@ -11,12 +11,12 @@ import { UpdateWorkspaceFeatureFlagInput } from 'src/engine/core-modules/admin-p
import { UserLookup } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.entity';
import { UserLookupInput } from 'src/engine/core-modules/admin-panel/dtos/user-lookup.input';
import { AuthGraphqlApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-graphql-api-exception.filter';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { ImpersonateGuard } from 'src/engine/guards/impersonate-guard';
import { UserAuthGuard } from 'src/engine/guards/user-auth.guard';
import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard';
import { AdminPanelHealthServiceData } from './dtos/admin-panel-health-service-data.dto';
import { AdminPanelIndicatorHealthStatusInputEnum } from './dtos/admin-panel-indicator-health-status.input';
@Resolver()
@UseFilters(AuthGraphqlApiExceptionFilter)
@ -70,11 +70,11 @@ export class AdminPanelResolver {
@Query(() => AdminPanelHealthServiceData)
async getIndicatorHealthStatus(
@Args('indicatorName', {
type: () => AdminPanelIndicatorHealthStatusInputEnum,
@Args('indicatorId', {
type: () => HealthIndicatorId,
})
indicatorName: AdminPanelIndicatorHealthStatusInputEnum,
indicatorId: HealthIndicatorId,
): Promise<AdminPanelHealthServiceData> {
return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorName);
return this.adminPanelHealthService.getIndicatorHealthStatus(indicatorId);
}
}

31
packages/twenty-server/src/engine/core-modules/admin-panel/constants/health-indicators.constants.ts Normal file
View File

@ -0,0 +1,31 @@
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
type HealthIndicatorInfo = {
id: HealthIndicatorId;
label: string;
description: string;
};
export const HEALTH_INDICATORS: Record<HealthIndicatorId, HealthIndicatorInfo> =
{
[HealthIndicatorId.database]: {
id: HealthIndicatorId.database,
label: 'Database Status',
description: 'PostgreSQL database connection status',
},
[HealthIndicatorId.redis]: {
id: HealthIndicatorId.redis,
label: 'Redis Status',
description: 'Redis connection status',
},
[HealthIndicatorId.worker]: {
id: HealthIndicatorId.worker,
label: 'Worker Status',
description: 'Background job worker status',
},
[HealthIndicatorId.connectedAccount]: {
id: HealthIndicatorId.connectedAccount,
label: 'Connected Account Status',
description: 'Connected accounts status',
},
};

8
packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-indicator.input.ts Normal file
View File

@ -0,0 +1,8 @@
import { Field } from '@nestjs/graphql';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
export class HealthIndicatorInput {
@Field(() => HealthIndicatorId)
indicatorId: HealthIndicatorId;
}

9
packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto.ts
View File

@ -5,6 +5,15 @@ import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-pan
@ObjectType()
export class AdminPanelHealthServiceData {
@Field(() => String)
id: string;
@Field(() => String)
label: string;
@Field(() => String)
description: string;
@Field(() => AdminPanelHealthServiceStatus)
status: AdminPanelHealthServiceStatus;

17
packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-indicator-health-status.input.ts
View File

@ -1,17 +0,0 @@
import { Field, registerEnumType } from '@nestjs/graphql';
export enum AdminPanelIndicatorHealthStatusInputEnum {
DATABASE = 'database',
REDIS = 'redis',
WORKER = 'worker',
MESSAGE_SYNC = 'messageSync',
}
registerEnumType(AdminPanelIndicatorHealthStatusInputEnum, {
name: 'AdminPanelIndicatorHealthStatusInputEnum',
});
export class AdminPanelIndicatorHealthStatusInput {
@Field(() => AdminPanelIndicatorHealthStatusInputEnum)
indicatorName: AdminPanelIndicatorHealthStatusInputEnum;
}

3
packages/twenty-server/src/engine/core-modules/admin-panel/dtos/admin-panel-worker-queue-health.dto.ts
View File

@ -5,6 +5,9 @@ import { WorkerQueueHealth } from 'src/engine/core-modules/health/types/worker-q
@ObjectType()
export class AdminPanelWorkerQueueHealth extends WorkerQueueHealth {
@Field(() => String)
id: string;
@Field(() => AdminPanelHealthServiceStatus)
status: AdminPanelHealthServiceStatus;
}

28
packages/twenty-server/src/engine/core-modules/admin-panel/dtos/system-health.dto.ts
View File

@ -1,18 +1,22 @@
import { Field, ObjectType } from '@nestjs/graphql';
import { AdminPanelHealthServiceData } from 'src/engine/core-modules/admin-panel/dtos/admin-panel-health-service-data.dto';
import { AdminPanelHealthServiceStatus } from 'src/engine/core-modules/admin-panel/enums/admin-panel-health-service-status.enum';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
@ObjectType()
export class SystemHealthService {
@Field(() => HealthIndicatorId)
id: HealthIndicatorId;
@Field(() => String)
label: string;
@Field(() => AdminPanelHealthServiceStatus)
status: AdminPanelHealthServiceStatus;
}
@ObjectType()
export class SystemHealth {
@Field(() => AdminPanelHealthServiceData)
database: AdminPanelHealthServiceData;
@Field(() => AdminPanelHealthServiceData)
redis: AdminPanelHealthServiceData;
@Field(() => AdminPanelHealthServiceData)
worker: AdminPanelHealthServiceData;
@Field(() => AdminPanelHealthServiceData)
messageSync: AdminPanelHealthServiceData;
@Field(() => [SystemHealthService])
services: SystemHealthService[];
}

3
packages/twenty-server/src/engine/core-modules/health/constants/health-error-messages.constants.ts
View File

@ -9,4 +9,7 @@ export const HEALTH_ERROR_MESSAGES = {
MESSAGE_SYNC_TIMEOUT: 'Message sync check timeout',
MESSAGE_SYNC_CHECK_FAILED: 'Message sync check failed',
MESSAGE_SYNC_HIGH_FAILURE_RATE: 'High failure rate in message sync jobs',
CALENDAR_SYNC_TIMEOUT: 'Calendar sync check timeout',
CALENDAR_SYNC_CHECK_FAILED: 'Calendar sync check failed',
CALENDAR_SYNC_HIGH_FAILURE_RATE: 'High failure rate in calendar sync jobs',
} as const;

1
packages/twenty-server/src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const.ts Normal file
View File

@ -0,0 +1 @@
export const METRICS_FAILURE_RATE_THRESHOLD = 20;

5
packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/health.controller.spec.ts
View File

@ -2,6 +2,7 @@ import { HealthCheckService } from '@nestjs/terminus';
import { Test, TestingModule } from '@nestjs/testing';
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
@ -29,6 +30,10 @@ describe('HealthController', () => {
provide: WorkerHealthIndicator,
useValue: { isHealthy: jest.fn() },
},
{
provide: ConnectedAccountHealth,
useValue: { isHealthy: jest.fn() },
},
],
}).compile();

1
packages/twenty-server/src/engine/core-modules/health/controllers/__tests__/metrics.controller.spec.ts
View File

@ -14,6 +14,7 @@ describe('MetricsController', () => {
provide: HealthCacheService,
useValue: {
getMessageChannelSyncJobByStatusCounter: jest.fn(),
getCalendarChannelSyncJobByStatusCounter: jest.fn(),
getInvalidCaptchaCounter: jest.fn(),
},
},

20
packages/twenty-server/src/engine/core-modules/health/controllers/health.controller.ts
View File

@ -1,7 +1,8 @@
import { BadRequestException, Controller, Get, Param } from '@nestjs/common';
import { HealthCheck, HealthCheckService } from '@nestjs/terminus';
import { HealthServiceName } from 'src/engine/core-modules/health/enums/health-service-name.enum';
import { HealthIndicatorId } from 'src/engine/core-modules/health/enums/health-indicator-id.enum';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { DatabaseHealthIndicator } from 'src/engine/core-modules/health/indicators/database.health';
import { RedisHealthIndicator } from 'src/engine/core-modules/health/indicators/redis.health';
import { WorkerHealthIndicator } from 'src/engine/core-modules/health/indicators/worker.health';
@ -13,6 +14,7 @@ export class HealthController {
private readonly databaseHealth: DatabaseHealthIndicator,
private readonly redisHealth: RedisHealthIndicator,
private readonly workerHealth: WorkerHealthIndicator,
private readonly connectedAccountHealth: ConnectedAccountHealth,
) {}
@Get()
@ -23,17 +25,19 @@ export class HealthController {
@Get('/:serviceName')
@HealthCheck()
checkService(@Param('serviceName') serviceName: HealthServiceName) {
checkService(@Param('indicatorId') indicatorId: HealthIndicatorId) {
const checks = {
[HealthServiceName.DATABASE]: () => this.databaseHealth.isHealthy(),
[HealthServiceName.REDIS]: () => this.redisHealth.isHealthy(),
[HealthServiceName.WORKER]: () => this.workerHealth.isHealthy(),
[HealthIndicatorId.database]: () => this.databaseHealth.isHealthy(),
[HealthIndicatorId.redis]: () => this.redisHealth.isHealthy(),
[HealthIndicatorId.worker]: () => this.workerHealth.isHealthy(),
[HealthIndicatorId.connectedAccount]: () =>
this.connectedAccountHealth.isHealthy(),
};
if (!(serviceName in checks)) {
throw new BadRequestException(`Invalid service name: ${serviceName}`);
if (!(indicatorId in checks)) {
throw new BadRequestException(`Invalid indicatorId: ${indicatorId}`);
}
return this.health.check([checks[serviceName]]);
return this.health.check([checks[indicatorId]]);
}
}

5
packages/twenty-server/src/engine/core-modules/health/controllers/metrics.controller.ts
View File

@ -15,4 +15,9 @@ export class MetricsController {
getInvalidCaptchaCounter() {
return this.healthCacheService.getInvalidCaptchaCounter();
}
@Get('/calendar-channel-sync-job-by-status-counter')
getCalendarChannelSyncJobByStatusCounter() {
return this.healthCacheService.getCalendarChannelSyncJobByStatusCounter();
}
}

12
packages/twenty-server/src/engine/core-modules/health/enums/health-indicator-id.enum.ts Normal file
View File

@ -0,0 +1,12 @@
import { registerEnumType } from '@nestjs/graphql';
export enum HealthIndicatorId {
database = 'database',
redis = 'redis',
worker = 'worker',
connectedAccount = 'connectedAccount',
}
registerEnumType(HealthIndicatorId, {
name: 'HealthIndicatorId',
});

6
packages/twenty-server/src/engine/core-modules/health/enums/health-service-name.enum.ts
View File

@ -1,6 +0,0 @@
export enum HealthServiceName {
DATABASE = 'database',
REDIS = 'redis',
WORKER = 'worker',
MESSAGE_SYNC = 'messageSync',
}

63
packages/twenty-server/src/engine/core-modules/health/health-cache.service.ts
View File

@ -4,8 +4,9 @@ import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decora
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
import { AccountSyncJobByStatusCounter } from 'src/engine/core-modules/health/types/account-sync-metrics.types';
import { HealthCounterCacheKeys } from 'src/engine/core-modules/health/types/health-counter-cache-keys.type';
import { MessageChannelSyncJobByStatusCounter } from 'src/engine/core-modules/health/types/message-sync-metrics.types';
import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
@Injectable()
@ -48,9 +49,7 @@ export class HealthCacheService {
);
const currentCounter =
await this.cacheStorage.get<MessageChannelSyncJobByStatusCounter>(
cacheKey,
);
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(cacheKey);
const updatedCounter = {
...(currentCounter || {}),
@ -80,7 +79,7 @@ export class HealthCacheService {
for (const key of cacheKeys) {
const counter =
await this.cacheStorage.get<MessageChannelSyncJobByStatusCounter>(key);
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(key);
if (!counter) continue;
@ -130,4 +129,58 @@ export class HealthCacheService {
return aggregatedCounter;
}
async incrementCalendarChannelSyncJobByStatusCounter(
status: CalendarChannelSyncStatus,
increment: number,
) {
const cacheKey = this.getCacheKeyWithTimestamp(
HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
);
const currentCounter =
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(cacheKey);
const updatedCounter = {
...(currentCounter || {}),
[status]: (currentCounter?.[status] || 0) + increment,
};
return await this.cacheStorage.set(
cacheKey,
updatedCounter,
this.healthCacheTtl,
);
}
async getCalendarChannelSyncJobByStatusCounter() {
const cacheKeys = this.getLastXMinutesTimestamps(
this.healthMonitoringTimeWindowInMinutes,
).map((timestamp) =>
this.getCacheKeyWithTimestamp(
HealthCounterCacheKeys.CalendarEventSyncJobByStatus,
timestamp,
),
);
const aggregatedCounter = Object.fromEntries(
Object.values(CalendarChannelSyncStatus).map((status) => [status, 0]),
);
for (const key of cacheKeys) {
const counter =
await this.cacheStorage.get<AccountSyncJobByStatusCounter>(key);
if (!counter) continue;
for (const [status, count] of Object.entries(counter) as [
CalendarChannelSyncStatus,
number,
][]) {
aggregatedCounter[status] += count;
}
}
return aggregatedCounter;
}
}

6
packages/twenty-server/src/engine/core-modules/health/health.module.ts
View File

@ -3,11 +3,11 @@ import { TerminusModule } from '@nestjs/terminus';
import { HealthController } from 'src/engine/core-modules/health/controllers/health.controller';
import { MetricsController } from 'src/engine/core-modules/health/controllers/metrics.controller';
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
import { RedisClientModule } from 'src/engine/core-modules/redis-client/redis-client.module';
import { HealthCacheService } from './health-cache.service';
import { ConnectedAccountHealth } from './indicators/connected-account.health';
import { DatabaseHealthIndicator } from './indicators/database.health';
import { RedisHealthIndicator } from './indicators/redis.health';
import { WorkerHealthIndicator } from './indicators/worker.health';
@ -19,14 +19,14 @@ import { WorkerHealthIndicator } from './indicators/worker.health';
DatabaseHealthIndicator,
RedisHealthIndicator,
WorkerHealthIndicator,
MessageSyncHealthIndicator,
ConnectedAccountHealth,
],
exports: [
HealthCacheService,
DatabaseHealthIndicator,
RedisHealthIndicator,
WorkerHealthIndicator,
MessageSyncHealthIndicator,
ConnectedAccountHealth,
],
})
export class HealthModule {}

316
packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/connected-account.health.spec.ts Normal file
View File

@ -0,0 +1,316 @@
import { HealthIndicatorService } from '@nestjs/terminus';
import { Test, TestingModule } from '@nestjs/testing';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts';
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const';
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
import { ConnectedAccountHealth } from 'src/engine/core-modules/health/indicators/connected-account.health';
import { CalendarChannelSyncStatus } from 'src/modules/calendar/common/standard-objects/calendar-channel.workspace-entity';
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
describe('ConnectedAccountHealth', () => {
let service: ConnectedAccountHealth;
let healthCacheService: jest.Mocked<HealthCacheService>;
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
beforeEach(async () => {
healthCacheService = {
getMessageChannelSyncJobByStatusCounter: jest.fn(),
getCalendarChannelSyncJobByStatusCounter: jest.fn(),
} as any;
healthIndicatorService = {
check: jest.fn().mockImplementation((key) => ({
up: jest.fn().mockImplementation((data) => ({
[key]: {
status: 'up',
details: data.details,
},
})),
down: jest.fn().mockImplementation((data) => ({
[key]: {
status: 'down',
error: data.error,
details: data.details,
},
})),
})),
} as any;
const module: TestingModule = await Test.createTestingModule({
providers: [
ConnectedAccountHealth,
{
provide: HealthCacheService,
useValue: healthCacheService,
},
{
provide: HealthIndicatorService,
useValue: healthIndicatorService,
},
],
}).compile();
service = module.get<ConnectedAccountHealth>(ConnectedAccountHealth);
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('message sync health', () => {
it('should return up status when no message sync jobs are present', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ONGOING]: 0,
[MessageChannelSyncStatus.ACTIVE]: 0,
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 0,
},
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
{
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
[CalendarChannelSyncStatus.ACTIVE]: 0,
},
);
const result = await service.isHealthy();
expect(result.connectedAccount.status).toBe('up');
expect(result.connectedAccount.details.messageSync.status).toBe('up');
expect(
result.connectedAccount.details.messageSync.details.totalJobs,
).toBe(0);
expect(
result.connectedAccount.details.messageSync.details.failedJobs,
).toBe(0);
expect(
result.connectedAccount.details.messageSync.details.failureRate,
).toBe(0);
});
it(`should return down status when message sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ONGOING]: 1,
[MessageChannelSyncStatus.ACTIVE]: 1,
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
},
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
{
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
[CalendarChannelSyncStatus.ACTIVE]: 1,
},
);
const result = await service.isHealthy();
expect(result.connectedAccount.status).toBe('down');
expect(result.connectedAccount.error).toBe(
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
);
expect(result.connectedAccount.details.messageSync.status).toBe('down');
expect(result.connectedAccount.details.messageSync.error).toBe(
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
);
expect(
result.connectedAccount.details.messageSync.details.failureRate,
).toBe(33.33);
});
});
describe('calendar sync health', () => {
it('should return up status when no calendar sync jobs are present', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ACTIVE]: 0,
},
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
{
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
[CalendarChannelSyncStatus.ONGOING]: 0,
[CalendarChannelSyncStatus.ACTIVE]: 0,
[CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 0,
},
);
const result = await service.isHealthy();
expect(result.connectedAccount.status).toBe('up');
expect(result.connectedAccount.details.calendarSync.status).toBe('up');
expect(
result.connectedAccount.details.calendarSync.details.totalJobs,
).toBe(0);
expect(
result.connectedAccount.details.calendarSync.details.failedJobs,
).toBe(0);
expect(
result.connectedAccount.details.calendarSync.details.failureRate,
).toBe(0);
});
it(`should return down status when calendar sync failure rate is above ${METRICS_FAILURE_RATE_THRESHOLD}%`, async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ACTIVE]: 1,
},
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
{
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
[CalendarChannelSyncStatus.ONGOING]: 1,
[CalendarChannelSyncStatus.ACTIVE]: 1,
[CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 2,
},
);
const result = await service.isHealthy();
expect(result.connectedAccount.status).toBe('down');
expect(result.connectedAccount.error).toBe(
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
);
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
expect(result.connectedAccount.details.calendarSync.error).toBe(
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
);
expect(
result.connectedAccount.details.calendarSync.details.failureRate,
).toBe(33.33);
});
});
describe('timeout handling', () => {
it('should handle message sync timeout', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockImplementationOnce(
() =>
new Promise((resolve) =>
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
),
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
{
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
[CalendarChannelSyncStatus.ACTIVE]: 1,
},
);
const healthCheckPromise = service.isHealthy();
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
const result = await healthCheckPromise;
expect(result.connectedAccount.status).toBe('down');
expect(result.connectedAccount.error).toBe(
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
);
expect(result.connectedAccount.details.messageSync.status).toBe('down');
expect(result.connectedAccount.details.messageSync.error).toBe(
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
);
});
it('should handle calendar sync timeout', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ACTIVE]: 1,
},
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockImplementationOnce(
() =>
new Promise((resolve) =>
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
),
);
const healthCheckPromise = service.isHealthy();
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
const result = await healthCheckPromise;
expect(result.connectedAccount.status).toBe('down');
expect(result.connectedAccount.error).toBe(
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
);
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
expect(result.connectedAccount.details.calendarSync.error).toBe(
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
);
});
});
describe('combined health check', () => {
it('should return combined status with both checks healthy', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ACTIVE]: 8,
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 1,
},
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
{
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
[CalendarChannelSyncStatus.ACTIVE]: 8,
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 1,
},
);
const result = await service.isHealthy();
expect(result.connectedAccount.status).toBe('up');
expect(result.connectedAccount.details.messageSync.status).toBe('up');
expect(result.connectedAccount.details.calendarSync.status).toBe('up');
});
it('should return down status when both syncs fail', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ACTIVE]: 1,
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
},
);
healthCacheService.getCalendarChannelSyncJobByStatusCounter.mockResolvedValue(
{
[CalendarChannelSyncStatus.NOT_SYNCED]: 0,
[CalendarChannelSyncStatus.ACTIVE]: 1,
[CalendarChannelSyncStatus.FAILED_UNKNOWN]: 2,
},
);
const result = await service.isHealthy();
expect(result.connectedAccount.status).toBe('down');
expect(result.connectedAccount.error).toBe(
`${HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE} and ${HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE}`,
);
expect(result.connectedAccount.details.messageSync.status).toBe('down');
expect(result.connectedAccount.details.calendarSync.status).toBe('down');
});
});
});

137
packages/twenty-server/src/engine/core-modules/health/indicators/__tests__/message-sync.health.spec.ts
View File

@ -1,137 +0,0 @@
import { HealthIndicatorService } from '@nestjs/terminus';
import { Test, TestingModule } from '@nestjs/testing';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/health/constants/health-indicators-timeout.conts';
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
import { MessageSyncHealthIndicator } from 'src/engine/core-modules/health/indicators/message-sync.health';
import { MessageChannelSyncStatus } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity';
describe('MessageSyncHealthIndicator', () => {
let service: MessageSyncHealthIndicator;
let healthCacheService: jest.Mocked<HealthCacheService>;
let healthIndicatorService: jest.Mocked<HealthIndicatorService>;
beforeEach(async () => {
healthCacheService = {
getMessageChannelSyncJobByStatusCounter: jest.fn(),
} as any;
healthIndicatorService = {
check: jest.fn().mockReturnValue({
up: jest.fn().mockImplementation((data) => ({
messageSync: { status: 'up', ...data },
})),
down: jest.fn().mockImplementation((error) => ({
messageSync: { status: 'down', error },
})),
}),
} as any;
const module: TestingModule = await Test.createTestingModule({
providers: [
MessageSyncHealthIndicator,
{
provide: HealthCacheService,
useValue: healthCacheService,
},
{
provide: HealthIndicatorService,
useValue: healthIndicatorService,
},
],
}).compile();
service = module.get<MessageSyncHealthIndicator>(
MessageSyncHealthIndicator,
);
jest.useFakeTimers();
});
afterEach(() => {
jest.useRealTimers();
});
it('should be defined', () => {
expect(service).toBeDefined();
});
it('should return up status when no jobs are present', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ONGOING]: 0,
[MessageChannelSyncStatus.ACTIVE]: 0,
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 0,
},
);
const result = await service.isHealthy();
expect(result.messageSync.status).toBe('up');
expect(result.messageSync.details.totalJobs).toBe(0);
expect(result.messageSync.details.failedJobs).toBe(0);
expect(result.messageSync.details.failureRate).toBe(0);
});
it('should return up status when failure rate is below 20%', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ONGOING]: 2,
[MessageChannelSyncStatus.ACTIVE]: 8,
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 0,
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 1,
},
);
const result = await service.isHealthy();
expect(result.messageSync.status).toBe('up');
expect(result.messageSync.details.totalJobs).toBe(11);
expect(result.messageSync.details.failedJobs).toBe(1);
expect(result.messageSync.details.failureRate).toBe(9.09);
});
it('should return down status when failure rate is above 20%', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockResolvedValue(
{
[MessageChannelSyncStatus.NOT_SYNCED]: 0,
[MessageChannelSyncStatus.ONGOING]: 1,
[MessageChannelSyncStatus.ACTIVE]: 1,
[MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS]: 2,
[MessageChannelSyncStatus.FAILED_UNKNOWN]: 2,
},
);
const result = await service.isHealthy();
expect(result.messageSync.status).toBe('down');
expect(result.messageSync.error.error).toBe(
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
);
expect(result.messageSync.error.details).toBeDefined();
expect(result.messageSync.error.details.failureRate).toBe(33.33);
});
it('should timeout after specified duration', async () => {
healthCacheService.getMessageChannelSyncJobByStatusCounter.mockImplementationOnce(
() =>
new Promise((resolve) =>
setTimeout(resolve, HEALTH_INDICATORS_TIMEOUT + 100),
),
);
const healthCheckPromise = service.isHealthy();
jest.advanceTimersByTime(HEALTH_INDICATORS_TIMEOUT + 1);
const result = await healthCheckPromise;
expect(result.messageSync.status).toBe('down');
expect(result.messageSync.error).toBe(
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
);
});
});

155
packages/twenty-server/src/engine/core-modules/health/indicators/connected-account.health.ts Normal file
View File

@ -0,0 +1,155 @@
import { Injectable } from '@nestjs/common';
import {
HealthIndicatorResult,
HealthIndicatorService,
} from '@nestjs/terminus';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { METRICS_FAILURE_RATE_THRESHOLD } from 'src/engine/core-modules/health/constants/metrics-failure-rate-threshold.const';
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
@Injectable()
export class ConnectedAccountHealth {
constructor(
private readonly healthIndicatorService: HealthIndicatorService,
private readonly healthCacheService: HealthCacheService,
) {}
private async checkMessageSyncHealth(): Promise<HealthIndicatorResult> {
const indicator = this.healthIndicatorService.check('messageSync');
try {
const counters = await withHealthCheckTimeout(
this.healthCacheService.getMessageChannelSyncJobByStatusCounter(),
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
);
const totalJobs = Object.values(counters).reduce(
(sum, count) => sum + (count || 0),
0,
);
const failedJobs = counters.FAILED_UNKNOWN || 0;
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
const failureRate =
totalJobs > 0
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
: 0;
const details = {
counters,
totalJobs,
failedJobs,
failureRate,
};
if (totalJobs === 0 || failureRate < METRICS_FAILURE_RATE_THRESHOLD) {
return indicator.up({ details });
}
return indicator.down({
error: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
details,
});
} catch (error) {
const errorMessage =
error.message === HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
? HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED;
return indicator.down({
error: errorMessage,
details: {},
});
}
}
private async checkCalendarSyncHealth(): Promise<HealthIndicatorResult> {
const indicator = this.healthIndicatorService.check('calendarSync');
try {
const counters = await withHealthCheckTimeout(
this.healthCacheService.getCalendarChannelSyncJobByStatusCounter(),
HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT,
);
const totalJobs = Object.values(counters).reduce(
(sum, count) => sum + (count || 0),
0,
);
const failedJobs = counters.FAILED_UNKNOWN || 0;
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
const failureRate =
totalJobs > 0
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
: 0;
const details = {
counters,
totalJobs,
failedJobs,
failureRate,
};
if (totalJobs === 0 || failureRate < METRICS_FAILURE_RATE_THRESHOLD) {
return indicator.up({ details });
}
return indicator.down({
error: HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_HIGH_FAILURE_RATE,
details,
});
} catch (error) {
const errorMessage =
error.message === HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT
? HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_TIMEOUT
: HEALTH_ERROR_MESSAGES.CALENDAR_SYNC_CHECK_FAILED;
return indicator.down({
error: errorMessage,
details: {},
});
}
}
async isHealthy(): Promise<HealthIndicatorResult> {
const indicator = this.healthIndicatorService.check('connectedAccount');
const [messageResult, calendarResult] = await Promise.all([
this.checkMessageSyncHealth(),
this.checkCalendarSyncHealth(),
]);
const isMessageSyncDown = messageResult.messageSync.status === 'down';
const isCalendarSyncDown = calendarResult.calendarSync.status === 'down';
if (isMessageSyncDown || isCalendarSyncDown) {
let error: string;
if (isMessageSyncDown && isCalendarSyncDown) {
error = `${messageResult.messageSync.error} and ${calendarResult.calendarSync.error}`;
} else if (isMessageSyncDown) {
error = messageResult.messageSync.error;
} else {
error = calendarResult.calendarSync.error;
}
return indicator.down({
error,
details: {
messageSync: messageResult.messageSync,
calendarSync: calendarResult.calendarSync,
},
});
}
return indicator.up({
details: {
messageSync: messageResult.messageSync,
calendarSync: calendarResult.calendarSync,
},
});
}
}

63
packages/twenty-server/src/engine/core-modules/health/indicators/message-sync.health.ts
View File

@ -1,63 +0,0 @@
import { Injectable } from '@nestjs/common';
import {
HealthIndicatorResult,
HealthIndicatorService,
} from '@nestjs/terminus';
import { HEALTH_ERROR_MESSAGES } from 'src/engine/core-modules/health/constants/health-error-messages.constants';
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
import { withHealthCheckTimeout } from 'src/engine/core-modules/health/utils/health-check-timeout.util';
@Injectable()
export class MessageSyncHealthIndicator {
constructor(
private readonly healthIndicatorService: HealthIndicatorService,
private readonly healthCacheService: HealthCacheService,
) {}
async isHealthy(): Promise<HealthIndicatorResult> {
const indicator = this.healthIndicatorService.check('messageSync');
try {
const counters = await withHealthCheckTimeout(
this.healthCacheService.getMessageChannelSyncJobByStatusCounter(),
HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT,
);
const totalJobs = Object.values(counters).reduce(
(sum, count) => sum + (count || 0),
0,
);
const failedJobs = counters.FAILED_UNKNOWN || 0;
// + (counters.FAILED_INSUFFICIENT_PERMISSIONS || 0)
const failureRate =
totalJobs > 0
? Math.round((failedJobs / totalJobs) * 100 * 100) / 100
: 0;
const details = {
counters,
totalJobs,
failedJobs,
failureRate,
};
if (totalJobs === 0 || failureRate < 20) {
return indicator.up({ details });
}
return indicator.down({
error: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_HIGH_FAILURE_RATE,
details,
});
} catch (error) {
const errorMessage =
error.message === HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
? HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_TIMEOUT
: HEALTH_ERROR_MESSAGES.MESSAGE_SYNC_CHECK_FAILED;
return indicator.down(errorMessage);
}
}
}

2
packages/twenty-server/src/engine/core-modules/health/indicators/worker.health.ts
View File

@ -71,7 +71,7 @@ export class WorkerHealthIndicator {
]);
queueStatuses.push({
name: queueName,
queueName: queueName,
workers: workers.length,
metrics: {
failed: failedCount,

2
packages/twenty-server/src/engine/core-modules/health/types/message-sync-metrics.types.ts → packages/twenty-server/src/engine/core-modules/health/types/account-sync-metrics.types.ts
View File

@ -1,7 +1,7 @@
import { Field, ObjectType } from '@nestjs/graphql';
@ObjectType()
export class MessageChannelSyncJobByStatusCounter {
export class AccountSyncJobByStatusCounter {
@Field(() => Number, { nullable: true })
NOT_SYNCED?: number;

1
packages/twenty-server/src/engine/core-modules/health/types/health-counter-cache-keys.type.ts
View File

@ -1,4 +1,5 @@
export enum HealthCounterCacheKeys {
MessageChannelSyncJobByStatus = 'message-channel-sync-job-by-status',
InvalidCaptcha = 'invalid-captcha',
CalendarEventSyncJobByStatus = 'calendar-event-sync-job-by-status',
}

2
packages/twenty-server/src/engine/core-modules/health/types/worker-queue-health.type.ts
View File

@ -5,7 +5,7 @@ import { WorkerQueueMetrics } from 'src/engine/core-modules/health/types/worker-
@ObjectType()
export class WorkerQueueHealth {
@Field(() => String)
name: string;
queueName: string;
@Field(() => Number)
workers: number;

3
packages/twenty-server/src/modules/calendar/calendar-event-import-manager/calendar-event-import-manager.module.ts
View File

@ -3,6 +3,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { HealthModule } from 'src/engine/core-modules/health/health.module';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
@ -47,9 +48,9 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
MicrosoftCalendarDriverModule,
BillingModule,
RefreshAccessTokenManagerModule,
CalendarEventParticipantManagerModule,
ConnectedAccountModule,
CalendarCommonModule,
HealthModule,
],
providers: [
CalendarChannelSyncStatusService,

2
packages/twenty-server/src/modules/calendar/common/calendar-common.module.ts
View File

@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { FeatureFlag } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { HealthModule } from 'src/engine/core-modules/health/health.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
@ -11,6 +12,7 @@ import { ConnectedAccountModule } from 'src/modules/connected-account/connected-
WorkspaceDataSourceModule,
TypeOrmModule.forFeature([FeatureFlag], 'core'),
ConnectedAccountModule,
HealthModule,
],
providers: [CalendarChannelSyncStatusService],
exports: [CalendarChannelSyncStatusService],

22
packages/twenty-server/src/modules/calendar/common/services/calendar-channel-sync-status.service.ts
View File

@ -5,6 +5,7 @@ import { Any } from 'typeorm';
import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
import { HealthCacheService } from 'src/engine/core-modules/health/health-cache.service';
import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
import {
CalendarChannelSyncStage,
@ -22,6 +23,7 @@ export class CalendarChannelSyncStatusService {
@InjectCacheStorage(CacheStorageNamespace.ModuleCalendar)
private readonly cacheStorage: CacheStorageService,
private readonly accountsToReconnectService: AccountsToReconnectService,
private readonly healthCacheService: HealthCacheService,
) {}
public async scheduleFullCalendarEventListFetch(
@ -77,6 +79,11 @@ export class CalendarChannelSyncStatusService {
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
});
await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
CalendarChannelSyncStatus.ONGOING,
calendarChannelIds.length,
);
}
public async resetAndScheduleFullCalendarEventListFetch(
@ -175,6 +182,11 @@ export class CalendarChannelSyncStatusService {
});
await this.schedulePartialCalendarEventListFetch(calendarChannelIds);
await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
CalendarChannelSyncStatus.ACTIVE,
calendarChannelIds.length,
);
}
public async markAsFailedUnknownAndFlushCalendarEventsToImport(
@ -200,6 +212,11 @@ export class CalendarChannelSyncStatusService {
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED,
});
await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
CalendarChannelSyncStatus.FAILED_UNKNOWN,
calendarChannelIds.length,
);
}
public async markAsFailedInsufficientPermissionsAndFlushCalendarEventsToImport(
@ -250,6 +267,11 @@ export class CalendarChannelSyncStatusService {
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
await this.healthCacheService.incrementCalendarChannelSyncJobByStatusCounter(
CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
calendarChannelIds.length,
);
}
private async addToAccountsToReconnect(