feat(analytics): add clickhouse (#11174)

This commit is contained in:
Antoine Moreaux
2025-04-16 18:33:10 +02:00
committed by GitHub
parent b6901a49bf
commit 587281a541
66 changed files with 1858 additions and 244 deletions

View File

@ -168,9 +168,25 @@ jobs:
image: redis
ports:
- 6379:6379
clickhouse:
image: clickhouse/clickhouse-server:latest
env:
CLICKHOUSE_PASSWORD: clickhousePassword
CLICKHOUSE_URL: "http://default:clickhousePassword@localhost:8123/twenty"
ports:
- 8123:8123
- 9000:9000
options: >-
--health-cmd "clickhouse-client --host=localhost --port=9000 --user=default --password=clickhousePassword --query='SELECT 1'"
--health-interval 10s
--health-timeout 5s
--health-retries 5
env:
NX_REJECT_UNKNOWN_LOCAL_CACHE: 0
NODE_ENV: test
ANALYTICS_ENABLED: true
CLICKHOUSE_URL: "http://default:clickhousePassword@localhost:8123/twenty"
CLICKHOUSE_PASSWORD: clickhousePassword
steps:
- name: Fetch custom Github Actions and base branch history
uses: actions/checkout@v4
@ -178,7 +194,7 @@ jobs:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/workflows/actions/yarn-install
- name: Update .env.test for billing
- name: Update .env.test for integrations tests
run: |
echo "IS_BILLING_ENABLED=true" >> .env.test
echo "BILLING_STRIPE_API_KEY=test-api-key" >> .env.test
@ -198,6 +214,10 @@ jobs:
- name: Server / Create Test DB
run: |
PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";'
- name: Run ClickHouse migrations
run: npx nx clickhouse:migrate twenty-server
- name: Run ClickHouse seeds
run: npx nx clickhouse:seed twenty-server
- name: Server / Run Integration Tests
uses: ./.github/workflows/actions/nx-affected
with:

View File

@ -17,4 +17,7 @@ postgres-on-docker:
-c "CREATE DATABASE \"test\" WITH OWNER postgres;"
redis-on-docker:
docker run -d --name twenty_redis -p 6379:6379 redis/redis-stack-server:latest
docker run -d --name twenty_redis -p 6379:6379 redis/redis-stack-server:latest
clickhouse-on-docker:
docker run -d --name twenty_clickhouse -p 8123:8123 -p 9000:9000 -e CLICKHOUSE_PASSWORD=devPassword clickhouse/clickhouse-server:latest

View File

@ -1,5 +1,3 @@
const path = require('path');
module.exports = {
extends: ['../../.eslintrc.react.cjs'],
ignorePatterns: [

View File

@ -63,6 +63,11 @@ export type Analytics = {
success: Scalars['Boolean']['output'];
};
export enum AnalyticsType {
PAGEVIEW = 'PAGEVIEW',
TRACK = 'TRACK'
}
export type ApiConfig = {
__typename?: 'ApiConfig';
mutationMaximumAffectedRecords: Scalars['Float']['output'];
@ -155,7 +160,8 @@ export type BillingEndTrialPeriodOutput = {
export type BillingMeteredProductUsageOutput = {
__typename?: 'BillingMeteredProductUsageOutput';
includedFreeQuantity: Scalars['Float']['output'];
freeTierQuantity: Scalars['Float']['output'];
freeTrialQuantity: Scalars['Float']['output'];
periodEnd: Scalars['DateTime']['output'];
periodStart: Scalars['DateTime']['output'];
productKey: BillingProductKey;
@ -474,6 +480,10 @@ export type CreateServerlessFunctionInput = {
};
export type CreateWorkflowVersionStepInput = {
/** Next step ID */
nextStepId?: InputMaybe<Scalars['String']['input']>;
/** Parent step ID */
parentStepId?: InputMaybe<Scalars['String']['input']>;
/** New step type */
stepType: Scalars['String']['input'];
/** Workflow version ID */
@ -598,6 +608,12 @@ export type FeatureFlag = {
workspaceId: Scalars['String']['output'];
};
export type FeatureFlagDto = {
__typename?: 'FeatureFlagDTO';
key: FeatureFlagKey;
value: Scalars['Boolean']['output'];
};
export enum FeatureFlagKey {
IsAirtableIntegrationEnabled = 'IsAirtableIntegrationEnabled',
IsAnalyticsV2Enabled = 'IsAnalyticsV2Enabled',
@ -970,8 +986,9 @@ export type Mutation = {
syncRemoteTable: RemoteTable;
syncRemoteTableSchemaChanges: RemoteTable;
track: Analytics;
trackAnalytics: Analytics;
unsyncRemoteTable: RemoteTable;
updateLabPublicFeatureFlag: FeatureFlag;
updateLabPublicFeatureFlag: FeatureFlagDto;
updateOneField: Field;
updateOneObject: Object;
updateOneRemoteServer: RemoteServer;
@ -1252,6 +1269,14 @@ export type MutationTrackArgs = {
};
export type MutationTrackAnalyticsArgs = {
event?: InputMaybe<Scalars['String']['input']>;
name?: InputMaybe<Scalars['String']['input']>;
properties?: InputMaybe<Scalars['JSON']['input']>;
type: AnalyticsType;
};
export type MutationUnsyncRemoteTableArgs = {
input: RemoteTableInput;
};
@ -2462,7 +2487,7 @@ export type Workspace = {
defaultRole?: Maybe<Role>;
deletedAt?: Maybe<Scalars['DateTime']['output']>;
displayName?: Maybe<Scalars['String']['output']>;
featureFlags?: Maybe<Array<FeatureFlag>>;
featureFlags?: Maybe<Array<FeatureFlagDto>>;
hasValidEnterpriseKey: Scalars['Boolean']['output'];
id: Scalars['UUID']['output'];
inviteHash?: Maybe<Scalars['String']['output']>;

View File

@ -55,6 +55,11 @@ export type Analytics = {
success: Scalars['Boolean'];
};
export enum AnalyticsType {
PAGEVIEW = 'PAGEVIEW',
TRACK = 'TRACK'
}
export type ApiConfig = {
__typename?: 'ApiConfig';
mutationMaximumAffectedRecords: Scalars['Float'];
@ -899,6 +904,7 @@ export type Mutation = {
submitFormStep: Scalars['Boolean'];
switchToYearlyInterval: BillingUpdateOutput;
track: Analytics;
trackAnalytics: Analytics;
updateLabPublicFeatureFlag: FeatureFlagDto;
updateOneField: Field;
updateOneObject: Object;
@ -1139,6 +1145,14 @@ export type MutationTrackArgs = {
};
export type MutationTrackAnalyticsArgs = {
event?: InputMaybe<Scalars['String']>;
name?: InputMaybe<Scalars['String']>;
properties?: InputMaybe<Scalars['JSON']>;
type: AnalyticsType;
};
export type MutationUpdateLabPublicFeatureFlagArgs = {
input: UpdateLabPublicFeatureFlagInput;
};
@ -2403,6 +2417,16 @@ export type GetTimelineThreadsFromPersonIdQueryVariables = Exact<{
export type GetTimelineThreadsFromPersonIdQuery = { __typename?: 'Query', getTimelineThreadsFromPersonId: { __typename?: 'TimelineThreadsWithTotal', totalNumberOfThreads: number, timelineThreads: Array<{ __typename?: 'TimelineThread', id: any, read: boolean, visibility: MessageChannelVisibility, lastMessageReceivedAt: string, lastMessageBody: string, subject: string, numberOfMessagesInThread: number, participantCount: number, firstParticipant: { __typename?: 'TimelineThreadParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string }, lastTwoParticipants: Array<{ __typename?: 'TimelineThreadParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string }> }> } };
export type TrackAnalyticsMutationVariables = Exact<{
type: AnalyticsType;
event?: InputMaybe<Scalars['String']>;
name?: InputMaybe<Scalars['String']>;
properties?: InputMaybe<Scalars['JSON']>;
}>;
export type TrackAnalyticsMutation = { __typename?: 'Mutation', trackAnalytics: { __typename?: 'Analytics', success: boolean } };
export type TrackMutationVariables = Exact<{
action: Scalars['String'];
payload: Scalars['JSON'];
@ -3326,6 +3350,42 @@ export function useGetTimelineThreadsFromPersonIdLazyQuery(baseOptions?: Apollo.
export type GetTimelineThreadsFromPersonIdQueryHookResult = ReturnType<typeof useGetTimelineThreadsFromPersonIdQuery>;
export type GetTimelineThreadsFromPersonIdLazyQueryHookResult = ReturnType<typeof useGetTimelineThreadsFromPersonIdLazyQuery>;
export type GetTimelineThreadsFromPersonIdQueryResult = Apollo.QueryResult<GetTimelineThreadsFromPersonIdQuery, GetTimelineThreadsFromPersonIdQueryVariables>;
export const TrackAnalyticsDocument = gql`
mutation TrackAnalytics($type: AnalyticsType!, $event: String, $name: String, $properties: JSON) {
trackAnalytics(type: $type, event: $event, name: $name, properties: $properties) {
success
}
}
`;
export type TrackAnalyticsMutationFn = Apollo.MutationFunction<TrackAnalyticsMutation, TrackAnalyticsMutationVariables>;
/**
* __useTrackAnalyticsMutation__
*
* To run a mutation, you first call `useTrackAnalyticsMutation` within a React component and pass it any options that fit your needs.
* When your component renders, `useTrackAnalyticsMutation` returns a tuple that includes:
* - A mutate function that you can call at any time to execute the mutation
* - An object with fields that represent the current status of the mutation's execution
*
* @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2;
*
* @example
* const [trackAnalyticsMutation, { data, loading, error }] = useTrackAnalyticsMutation({
* variables: {
* type: // value for 'type'
* event: // value for 'event'
* name: // value for 'name'
* properties: // value for 'properties'
* },
* });
*/
export function useTrackAnalyticsMutation(baseOptions?: Apollo.MutationHookOptions<TrackAnalyticsMutation, TrackAnalyticsMutationVariables>) {
const options = {...defaultOptions, ...baseOptions}
return Apollo.useMutation<TrackAnalyticsMutation, TrackAnalyticsMutationVariables>(TrackAnalyticsDocument, options);
}
export type TrackAnalyticsMutationHookResult = ReturnType<typeof useTrackAnalyticsMutation>;
export type TrackAnalyticsMutationResult = Apollo.MutationResult<TrackAnalyticsMutation>;
export type TrackAnalyticsMutationOptions = Apollo.BaseMutationOptions<TrackAnalyticsMutation, TrackAnalyticsMutationVariables>;
export const TrackDocument = gql`
mutation Track($action: String!, $payload: JSON!) {
track(action: $action, payload: $payload) {

View File

@ -1,8 +1,18 @@
import { gql } from '@apollo/client';
export const TRACK = gql`
mutation Track($action: String!, $payload: JSON!) {
track(action: $action, payload: $payload) {
export const TRACK_ANALYTICS = gql`
mutation TrackAnalytics(
$type: AnalyticsType!
$event: String
$name: String
$properties: JSON
) {
trackAnalytics(
type: $type
event: $event
name: $name
properties: $properties
) {
success
}
}

View File

@ -5,23 +5,76 @@ import { act, renderHook, waitFor } from '@testing-library/react';
import { ReactNode } from 'react';
import { RecoilRoot } from 'recoil';
import { useEventTracker } from '../useEventTracker';
import { ANALYTICS_COOKIE_NAME, useEventTracker } from '../useEventTracker';
import { AnalyticsType } from '~/generated/graphql';
// Mock document.cookie
Object.defineProperty(document, 'cookie', {
writable: true,
value: `${ANALYTICS_COOKIE_NAME}=exampleId`,
});
const mocks: MockedResponse[] = [
{
request: {
query: gql`
mutation Track($action: String!, $payload: JSON!) {
track(action: $action, payload: $payload) {
mutation TrackAnalytics(
$type: AnalyticsType!
$event: String
$name: String
$properties: JSON
) {
trackAnalytics(
type: $type
event: $event
name: $name
properties: $properties
) {
success
}
}
`,
variables: {
action: 'exampleType',
payload: {
type: AnalyticsType['TRACK'],
event: 'Example Event',
properties: {
foo: 'bar',
},
},
},
result: jest.fn(() => ({
data: {
track: {
success: true,
},
},
})),
},
{
request: {
query: gql`
mutation TrackAnalytics(
$type: AnalyticsType!
$event: String
$name: String
$properties: JSON
) {
trackAnalytics(
type: $type
event: $event
name: $name
properties: $properties
) {
success
}
}
`,
variables: {
type: AnalyticsType['PAGEVIEW'],
name: 'Example',
properties: {
sessionId: 'exampleId',
pathname: '',
pathname: '/example/path',
userAgent: '',
timeZone: '',
locale: '',
@ -50,24 +103,45 @@ const Wrapper = ({ children }: { children: ReactNode }) => (
describe('useEventTracker', () => {
it('should make the call to track the event', async () => {
const eventType = 'exampleType';
const eventData = {
sessionId: 'exampleId',
pathname: '',
userAgent: '',
timeZone: '',
locale: '',
href: '',
referrer: '',
const payload = {
event: 'Example Event',
properties: {
foo: 'bar',
},
};
const { result } = renderHook(() => useEventTracker(), {
wrapper: Wrapper,
});
act(() => {
result.current(eventType, eventData);
result.current(AnalyticsType['TRACK'], payload);
});
await waitFor(() => {
expect(mocks[0].result).toHaveBeenCalled();
});
});
it('should make the call to track a pageview', async () => {
const payload = {
name: 'Example',
properties: {
sessionId: 'exampleId',
pathname: '/example/path',
userAgent: '',
timeZone: '',
locale: '',
href: '',
referrer: '',
},
};
const { result } = renderHook(() => useEventTracker(), {
wrapper: Wrapper,
});
act(() => {
result.current(AnalyticsType['PAGEVIEW'], payload);
});
await waitFor(() => {
expect(mocks[1].result).toHaveBeenCalled();
});
});
});

View File

@ -1,14 +1,11 @@
import { useCallback } from 'react';
import { v4 } from 'uuid';
import { useTrackMutation } from '~/generated/graphql';
export interface EventData {
pathname: string;
userAgent: string;
timeZone: string;
locale: string;
href: string;
referrer: string;
}
import {
AnalyticsType,
MutationTrackAnalyticsArgs,
useTrackAnalyticsMutation,
} from '~/generated/graphql';
export const ANALYTICS_COOKIE_NAME = 'analyticsCookie';
export const getSessionId = (): string => {
const cookie: { [key: string]: string } = {};
@ -28,16 +25,22 @@ export const setSessionId = (domain?: string): void => {
};
export const useEventTracker = () => {
const [createEventMutation] = useTrackMutation();
const [createEventMutation] = useTrackAnalyticsMutation();
return useCallback(
(eventAction: string, eventPayload: EventData) => {
(
type: AnalyticsType,
payload: Omit<MutationTrackAnalyticsArgs, 'type'>,
) => {
createEventMutation({
variables: {
action: eventAction,
payload: {
sessionId: getSessionId(),
...eventPayload,
type,
...payload,
properties: {
...payload.properties,
...(type === AnalyticsType['PAGEVIEW']
? { sessionId: getSessionId() }
: {}),
},
},
});

View File

@ -27,6 +27,8 @@ import { isDefined } from 'twenty-shared/utils';
import { useIsMatchingLocation } from '~/hooks/useIsMatchingLocation';
import { usePageChangeEffectNavigateLocation } from '~/hooks/usePageChangeEffectNavigateLocation';
import { useInitializeQueryParamState } from '~/modules/app/hooks/useInitializeQueryParamState';
import { AnalyticsType } from '~/generated/graphql';
import { getPageTitleFromPath } from '~/utils/title-utils';
// TODO: break down into smaller functions and / or hooks
// - moved usePageChangeEffectNavigateLocation into dedicated hook
@ -174,13 +176,16 @@ export const PageChangeEffect = () => {
useEffect(() => {
setTimeout(() => {
setSessionId();
eventTracker('pageview', {
pathname: location.pathname,
locale: navigator.language,
userAgent: window.navigator.userAgent,
href: window.location.href,
referrer: document.referrer,
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
eventTracker(AnalyticsType['PAGEVIEW'], {
name: getPageTitleFromPath(location.pathname),
properties: {
pathname: location.pathname,
locale: navigator.language,
userAgent: window.navigator.userAgent,
href: window.location.href,
referrer: document.referrer,
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
},
});
}, 500);
}, [eventTracker, location.pathname]);

View File

@ -1,7 +1,7 @@
import { getOperationName } from '@apollo/client/utilities';
import { graphql, GraphQLQuery, http, HttpResponse } from 'msw';
import { TRACK } from '@/analytics/graphql/queries/track';
import { TRACK_ANALYTICS } from '@/analytics/graphql/queries/track';
import { GET_CLIENT_CONFIG } from '@/client-config/graphql/queries/getClientConfig';
import { FIND_MANY_OBJECT_METADATA_ITEMS } from '@/object-metadata/graphql/queries';
import { GET_CURRENT_USER } from '@/users/graphql/queries/getCurrentUser';
@ -110,10 +110,10 @@ export const graphqlMocks = {
});
},
),
graphql.mutation(getOperationName(TRACK) ?? '', () => {
graphql.mutation(getOperationName(TRACK_ANALYTICS) ?? '', () => {
return HttpResponse.json({
data: {
track: { success: 1, __typename: 'TRACK' },
track: { success: 1, __typename: 'TRACK_ANALYTICS' },
},
});
}),

View File

@ -77,4 +77,5 @@ FRONTEND_URL=http://localhost:3001
# CLOUDFLARE_ZONE_ID=
# CLOUDFLARE_WEBHOOK_SECRET=
# IS_CONFIG_VARIABLES_IN_DB_ENABLED=false
# ANALYTICS_ENABLED=
# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty

View File

@ -21,3 +21,5 @@ MESSAGING_PROVIDER_GMAIL_CALLBACK_URL=http://localhost:3000/auth/google-gmail/ge
AUTH_MICROSOFT_CALLBACK_URL=http://localhost:3000/auth/microsoft/redirect
AUTH_MICROSOFT_APIS_CALLBACK_URL=http://localhost:3000/auth/microsoft-apis/get-access-token
CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty

View File

@ -15,6 +15,7 @@
"typeorm": "../../node_modules/typeorm/.bin/typeorm"
},
"dependencies": {
"@clickhouse/client": "^1.11.0",
"@esbuild-plugins/node-modules-polyfill": "^0.2.2",
"@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch",
"@langchain/mistralai": "^0.0.24",

View File

@ -204,6 +204,20 @@
},
"defaultConfiguration": "seed"
},
"clickhouse:migrate": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "nx ts-node-no-deps -- src/database/clickhouse/migrations/run-migrations.ts"
}
},
"clickhouse:seed": {
"executor": "nx:run-commands",
"options": {
"cwd": "packages/twenty-server",
"command": "nx ts-node-no-deps -- src/database/clickhouse/seeds/run-seeds.ts"
}
},
"lingui:extract": {
"executor": "nx:run-commands",
"options": {

View File

@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS events
(
`event` LowCardinality(String),
`timestamp` DateTime64(3),
`userId` String DEFAULT '',
`workspaceId` String DEFAULT '',
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (event, workspaceId, timestamp);

View File

@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS pageview
(
`name` LowCardinality(String),
`timestamp` DateTime64(3),
`properties` JSON,
`userId` String DEFAULT '',
`workspaceId` String DEFAULT ''
)
ENGINE = MergeTree
ORDER BY (name, workspaceId, userId, timestamp);

View File

@ -0,0 +1,107 @@
/* eslint-disable no-console */
import fs from 'fs';
import path from 'path';
import { createClient, ClickHouseClient } from '@clickhouse/client';
import { config } from 'dotenv';
config({
path: process.env.NODE_ENV === 'test' ? '.env.test' : '.env',
override: true,
});
const clickhouseUrl = () => {
const url = process.env.CLICKHOUSE_URL;
if (url) return url;
throw new Error(
'CLICKHOUSE_URL environment variable is not set. Please set it to the ClickHouse URL.',
);
};
async function ensureDatabaseExists() {
const [url, database] = clickhouseUrl().split(/\/(?=[^/]*$)/);
const client = createClient({
url,
});
await client.command({
query: `CREATE DATABASE IF NOT EXISTS "${database}"`,
});
await client.command({
query: `SET enable_json_type = 1`,
});
await client.close();
}
async function ensureMigrationTable(client: ClickHouseClient) {
await client.command({
query: `
CREATE TABLE IF NOT EXISTS migrations (
filename String,
applied_at DateTime DEFAULT now()
) ENGINE = MergeTree()
ORDER BY filename;
`,
});
}
async function hasMigrationBeenRun(
filename: string,
client: ClickHouseClient,
): Promise<boolean> {
const resultSet = await client.query({
query: `SELECT count() as count FROM migrations WHERE filename = {filename:String}`,
query_params: { filename },
format: 'JSON',
});
const result = await resultSet.json<{ count: number }>();
return result.data[0].count > 0;
}
async function recordMigration(filename: string, client: ClickHouseClient) {
await client.insert({
table: 'migrations',
values: [{ filename }],
format: 'JSONEachRow',
});
}
async function runMigrations() {
const dir = path.join(__dirname);
const files = fs.readdirSync(dir).filter((f) => f.endsWith('.sql'));
await ensureDatabaseExists();
const client = createClient({
url: clickhouseUrl(),
});
await ensureMigrationTable(client);
for (const file of files) {
const alreadyRun = await hasMigrationBeenRun(file, client);
if (alreadyRun) {
console.log(`✔︎ Skipping already applied migration: ${file}`);
continue;
}
const sql = fs.readFileSync(path.join(dir, file), 'utf8');
console.log(`⚡ Running ${file}...`);
await client.command({ query: sql });
await recordMigration(file, client);
}
console.log('✅ All migrations applied.');
await client.close();
}
runMigrations().catch((err) => {
console.error('Migration error:', err);
process.exit(1);
});

View File

@ -0,0 +1,38 @@
/* eslint-disable no-console */
import { createClient } from '@clickhouse/client';
import { config } from 'dotenv';
import { fixtures } from 'src/engine/core-modules/analytics/utils/fixtures/fixtures';
config({
path: process.env.NODE_ENV === 'test' ? '.env.test' : '.env',
override: true,
});
const client = createClient({
url: process.env.CLICKHOUSE_URL,
});
async function seedEvents() {
try {
console.log(`⚡ Seeding ${fixtures.length} events...`);
await client.insert({
table: 'events',
values: fixtures,
format: 'JSONEachRow',
});
console.log('✅ All events seeded successfully');
} catch (error) {
console.error('Error seeding events:', error);
throw error;
} finally {
await client.close();
}
}
seedEvents().catch((err) => {
console.error('Seeding error:', err);
process.exit(1);
});

View File

@ -1,13 +1,12 @@
import { Injectable } from '@nestjs/common';
import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { TelemetryService } from 'src/engine/core-modules/telemetry/telemetry.service';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event.type';
import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants';
import { OnCustomBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-custom-batch-event.decorator';
import { USER_SIGNUP_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/user/user-signup';
import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants';
@Injectable()
export class TelemetryListener {
@ -16,36 +15,18 @@ export class TelemetryListener {
private readonly telemetryService: TelemetryService,
) {}
@OnDatabaseBatchEvent('*', DatabaseEventAction.CREATED)
async handleAllCreate(payload: WorkspaceEventBatch<ObjectRecordCreateEvent>) {
await Promise.all(
payload.events.map((eventPayload) =>
this.analyticsService.create(
{
action: payload.name,
payload: {},
},
eventPayload.userId,
payload.workspaceId,
),
),
);
}
@OnCustomBatchEvent(USER_SIGNUP_EVENT_NAME)
async handleUserSignup(
payload: WorkspaceEventBatch<ObjectRecordCreateEvent>,
) {
await Promise.all(
payload.events.map(async (eventPayload) => {
this.analyticsService.create(
{
action: USER_SIGNUP_EVENT_NAME,
payload: {},
},
eventPayload.userId,
payload.workspaceId,
);
this.analyticsService
.createAnalyticsContext({
userId: eventPayload.userId,
workspaceId: payload.workspaceId,
})
.track(USER_SIGNUP_EVENT, {});
this.telemetryService.create(
{

View File

@ -0,0 +1,143 @@
# Analytics Module
This module provides analytics tracking functionality for the Twenty application.
## Usage
### Tracking Events
The `AnalyticsService` provides a `createAnalyticsContext` method that returns an object with a `track` method. The `track` method is used to track events.
```typescript
import { Injectable } from '@nestjs/common';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated';
@Injectable()
export class MyService {
constructor(private readonly analyticsService: AnalyticsService) {}
async doSomething() {
// Create an analytics context
const analytics = this.analyticsService.createAnalyticsContext({
workspaceId: 'workspace-id',
userId: 'user-id',
});
// Track an event
// The event name will be autocompleted
// The properties will be type-checked based on the event name
analytics.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
}
}
```
### Adding New Events
To add a new event:
1. Create a new file in the `src/engine/core-modules/analytics/utils/events/track` directory
2. Define the event name, schema, and type
3. Register the event using the `registerEvent` function
4. Update the `TrackEventName` and `TrackEventProperties` types in `src/engine/core-modules/analytics/utils/events/event-types.ts`
Example:
```typescript
// src/engine/core-modules/analytics/utils/events/track/my-feature/my-event.ts
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const MY_EVENT = 'My Event' as const;
export const myEventSchema = z.object({
event: z.literal(MY_EVENT),
properties: z.object({
myProperty: z.string(),
}),
});
export type MyEventTrackEvent = z.infer<typeof myEventSchema>;
registerEvent(MY_EVENT, myEventSchema);
```
Then update the `events.type.ts` file:
```typescript
// src/engine/core-modules/analytics/types/events.type.ts
import { MY_EVENT, MyEventTrackEvent } from '../utils/events/track/my-feature/my-event';
// Add to the union type
export type TrackEventName =
| typeof MY_EVENT
// ... other event names;
// Add to the TrackEvents interface
export interface TrackEvents {
[MY_EVENT]: MyEventTrackEvent;
// ... other event types
}
// The TrackEventProperties type will automatically use the new event
export type TrackEventProperties<T extends TrackEventName> = T extends keyof TrackEvents
? TrackEvents[T]['properties']
: object;
```
## API
### AnalyticsService
#### createAnalyticsContext(context?)
Creates an analytics context with the given user ID and workspace ID.
- `context` (optional): An object with `userId` and `workspaceId` properties
Returns an object with the following methods:
- `track<T extends TrackEventName>(event: T, properties: TrackEventProperties<T>)`: Tracks an event with the given name and properties
- `pageview(name: string, properties: Partial<PageviewProperties>)`: Tracks a pageview with the given name and properties
### Types
#### TrackEventName
A union type of all registered event names, plus `string` for backward compatibility.
#### TrackEventProperties<T>
A mapped type that maps each event name to its corresponding properties type. It uses the `TrackEvents` interface to provide a more maintainable and type-safe way to map event names to their properties.
```typescript
// Define the mapping between event names and their event types
export interface TrackEvents {
[EVENT_NAME_1]: Event1Type;
[EVENT_NAME_2]: Event2Type;
// ... other event types
}
// Use the mapping to extract properties for each event type
export type TrackEventProperties<T extends TrackEventName> = T extends keyof TrackEvents
? TrackEvents[T]['properties']
: object;
```
This approach makes it easier to add new events without having to modify a complex nested conditional type.
#### PageviewProperties
A type that defines the structure of pageview properties:
```typescript
type PageviewProperties = {
href: string;
locale: string;
pathname: string;
referrer: string;
sessionId: string;
timeZone: string;
userAgent: string;
};
```

View File

@ -0,0 +1,12 @@
import { CustomException } from 'src/utils/custom-exception';
export class AnalyticsException extends CustomException {
constructor(message: string, code: AnalyticsExceptionCode) {
super(message, code);
}
}
export enum AnalyticsExceptionCode {
INVALID_TYPE = 'INVALID_TYPE',
INVALID_INPUT = 'INVALID_INPUT',
}

View File

@ -1,12 +1,14 @@
import { Module } from '@nestjs/common';
import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module';
import { ClickhouseService } from 'src/engine/core-modules/analytics/services/clickhouse.service';
import { AnalyticsResolver } from './analytics.resolver';
import { AnalyticsService } from './analytics.service';
import { AnalyticsService } from './services/analytics.service';
@Module({
providers: [AnalyticsResolver, AnalyticsService],
providers: [AnalyticsResolver, AnalyticsService, ClickhouseService],
imports: [JwtModule],
exports: [AnalyticsService],
})

View File

@ -1,18 +1,31 @@
import { Test, TestingModule } from '@nestjs/testing';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { User } from 'src/engine/core-modules/user/user.entity';
import {
AnalyticsException,
AnalyticsExceptionCode,
} from 'src/engine/core-modules/analytics/analytics.exception';
import { AnalyticsResolver } from './analytics.resolver';
import { AnalyticsService } from './analytics.service';
import { AnalyticsService } from './services/analytics.service';
describe('AnalyticsResolver', () => {
let resolver: AnalyticsResolver;
let analyticsService: jest.Mocked<AnalyticsService>;
beforeEach(async () => {
analyticsService = {
createAnalyticsContext: jest.fn(),
} as any;
const module: TestingModule = await Test.createTestingModule({
providers: [
AnalyticsResolver,
{
provide: AnalyticsService,
useValue: {},
useValue: analyticsService,
},
],
}).compile();
@ -23,4 +36,71 @@ describe('AnalyticsResolver', () => {
it('should be defined', () => {
expect(resolver).toBeDefined();
});
it('should handle a valid pageview input', async () => {
const mockPageview = jest.fn().mockResolvedValue('Pageview created');
analyticsService.createAnalyticsContext.mockReturnValue({
pageview: mockPageview,
track: jest.fn(),
});
const input = {
type: 'pageview' as const,
name: 'Test Page',
properties: {},
};
const result = await resolver.trackAnalytics(
input,
{ id: 'workspace-1' } as Workspace,
{ id: 'user-1' } as User,
);
expect(analyticsService.createAnalyticsContext).toHaveBeenCalledWith({
workspaceId: 'workspace-1',
userId: 'user-1',
});
expect(mockPageview).toHaveBeenCalledWith('Test Page', {});
expect(result).toBe('Pageview created');
});
it('should handle a valid track input', async () => {
const mockTrack = jest.fn().mockResolvedValue('Track created');
analyticsService.createAnalyticsContext.mockReturnValue({
track: mockTrack,
pageview: jest.fn(),
});
const input = {
type: 'track' as const,
event: 'Custom Domain Activated' as const,
properties: {},
};
const result = await resolver.trackAnalytics(
input,
{ id: 'workspace-2' } as Workspace,
{ id: 'user-2' } as User,
);
expect(analyticsService.createAnalyticsContext).toHaveBeenCalledWith({
workspaceId: 'workspace-2',
userId: 'user-2',
});
expect(mockTrack).toHaveBeenCalledWith('Custom Domain Activated', {});
expect(result).toBe('Track created');
});
it('should throw an AnalyticsException for invalid input', async () => {
const invalidInput = { type: 'invalid' };
await expect(
resolver.trackAnalytics(invalidInput as any, undefined, undefined),
).rejects.toThrowError(
new AnalyticsException(
'Invalid analytics input',
AnalyticsExceptionCode.INVALID_TYPE,
),
);
});
});

View File

@ -4,26 +4,63 @@ import { User } from 'src/engine/core-modules/user/user.entity';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator';
import {
AnalyticsException,
AnalyticsExceptionCode,
} from 'src/engine/core-modules/analytics/analytics.exception';
import { AnalyticsService } from './analytics.service';
import { CreateAnalyticsInput } from './dtos/create-analytics.input';
import { AnalyticsService } from './services/analytics.service';
import {
CreateAnalyticsInput,
CreateAnalyticsInputV2,
isPageviewAnalyticsInput,
isTrackAnalyticsInput,
} from './dtos/create-analytics.input';
import { Analytics } from './entities/analytics.entity';
@Resolver(() => Analytics)
export class AnalyticsResolver {
constructor(private readonly analyticsService: AnalyticsService) {}
// deprecated
@Mutation(() => Analytics)
track(
@Args() createAnalyticsInput: CreateAnalyticsInput,
@Args() _createAnalyticsInput: CreateAnalyticsInput,
@AuthWorkspace() _workspace: Workspace | undefined,
@AuthUser({ allowUndefined: true }) _user: User | undefined,
) {
return { success: true };
}
@Mutation(() => Analytics)
async trackAnalytics(
@Args()
createAnalyticsInput: CreateAnalyticsInputV2,
@AuthWorkspace() workspace: Workspace | undefined,
@AuthUser({ allowUndefined: true }) user: User | undefined,
) {
return this.analyticsService.create(
createAnalyticsInput,
user?.id,
workspace?.id,
const analyticsContext = this.analyticsService.createAnalyticsContext({
workspaceId: workspace?.id,
userId: user?.id,
});
if (isPageviewAnalyticsInput(createAnalyticsInput)) {
return analyticsContext.pageview(
createAnalyticsInput.name,
createAnalyticsInput.properties ?? {},
);
}
if (isTrackAnalyticsInput(createAnalyticsInput)) {
return analyticsContext.track(
createAnalyticsInput.event,
createAnalyticsInput.properties ?? {},
);
}
throw new AnalyticsException(
'Invalid analytics input',
AnalyticsExceptionCode.INVALID_TYPE,
);
}
}

View File

@ -1,37 +0,0 @@
import { HttpService } from '@nestjs/axios';
import { Test, TestingModule } from '@nestjs/testing';
import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { AnalyticsService } from './analytics.service';
describe('AnalyticsService', () => {
let service: AnalyticsService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
AnalyticsService,
{
provide: TwentyConfigService,
useValue: {},
},
{
provide: JwtWrapperService,
useValue: {},
},
{
provide: HttpService,
useValue: {},
},
],
}).compile();
service = module.get<AnalyticsService>(AnalyticsService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -1,53 +0,0 @@
import { Injectable } from '@nestjs/common';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
type CreateEventInput = {
action: string;
payload: object;
};
@Injectable()
export class AnalyticsService {
constructor(private readonly twentyConfigService: TwentyConfigService) {}
async create(
createEventInput: CreateEventInput,
userId: string | null | undefined,
workspaceId: string | null | undefined,
) {
if (!this.twentyConfigService.get('ANALYTICS_ENABLED')) {
return { success: true };
}
let _data;
switch (createEventInput.action) {
case 'pageview':
_data = {
timestamp: new Date().toISOString(),
version: '1',
userId: userId,
workspaceId: workspaceId,
...createEventInput.payload,
};
break;
default:
_data = {
action: createEventInput.action,
timestamp: new Date().toISOString(),
version: '1',
userId: userId,
workspaceId: workspaceId,
payload: {
...createEventInput.payload,
},
};
break;
}
// TODO: send event to clickhouse
return { success: true };
}
}

View File

@ -1,8 +1,27 @@
import { ArgsType, Field } from '@nestjs/graphql';
import { ArgsType, Field, registerEnumType } from '@nestjs/graphql';
import { IsNotEmpty, IsObject, IsString } from 'class-validator';
import graphqlTypeJson from 'graphql-type-json';
import {
IsEnum,
IsNotEmpty,
IsObject,
IsOptional,
IsString,
} from 'class-validator';
import GraphQLJSON from 'graphql-type-json';
import { TrackEventName } from 'src/engine/core-modules/analytics/types/events.type';
import { PageviewProperties } from 'src/engine/core-modules/analytics/utils/events/pageview/pageview';
enum AnalyticsType {
PAGEVIEW = 'pageview',
TRACK = 'track',
}
registerEnumType(AnalyticsType, {
name: 'AnalyticsType',
});
// deprecated
@ArgsType()
export class CreateAnalyticsInput {
@Field({ description: 'Type of the event' })
@ -10,7 +29,41 @@ export class CreateAnalyticsInput {
@IsString()
action: string;
@Field(() => graphqlTypeJson, { description: 'Event payload in JSON format' })
@Field(() => GraphQLJSON, { description: 'Event payload in JSON format' })
@IsObject()
payload: JSON;
}
@ArgsType()
export class CreateAnalyticsInputV2 {
@Field(() => AnalyticsType)
@IsEnum(AnalyticsType)
type: 'pageview' | 'track';
@Field(() => String, { nullable: true })
@IsOptional()
@IsString()
name?: string;
@Field(() => String, { nullable: true })
@IsOptional()
@IsString()
event?: TrackEventName;
@Field(() => GraphQLJSON, { nullable: true })
@IsOptional()
@IsObject()
properties?: PageviewProperties | Record<string, unknown>;
}
export function isPageviewAnalyticsInput(
input: CreateAnalyticsInputV2,
): input is CreateAnalyticsInputV2 & { name: string } {
return input.type === 'pageview' && !!input.name;
}
export function isTrackAnalyticsInput(
input: CreateAnalyticsInputV2,
): input is CreateAnalyticsInputV2 & { event: TrackEventName } {
return input.type === 'track' && !!input.event;
}

View File

@ -0,0 +1,127 @@
import { Test, TestingModule } from '@nestjs/testing';
import { AnalyticsContextMock } from 'test/utils/analytics-context.mock';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { ClickhouseService } from 'src/engine/core-modules/analytics/services/clickhouse.service';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated';
import { AnalyticsService } from './analytics.service';
describe('AnalyticsService', () => {
let service: AnalyticsService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
{
provide: AnalyticsService,
useValue: {
createAnalyticsContext: AnalyticsContextMock,
},
},
{
provide: TwentyConfigService,
useValue: {
get: jest.fn().mockReturnValue(true),
},
},
{
provide: ClickhouseService,
useValue: {
pushEvent: jest.fn(),
},
},
{
provide: ExceptionHandlerService,
useValue: {
captureExceptions: jest.fn(),
},
},
],
}).compile();
service = module.get<AnalyticsService>(AnalyticsService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('createAnalyticsContext', () => {
const mockUserIdAndWorkspaceId = {
userId: 'test-user-id',
workspaceId: 'test-workspace-id',
};
it('should create a valid context object', () => {
const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId);
expect(context).toHaveProperty('track');
expect(context).toHaveProperty('pageview');
});
it('should call track with correct parameters', async () => {
const trackSpy = jest.fn().mockResolvedValue({ success: true });
const mockContext = AnalyticsContextMock({
track: trackSpy,
});
jest
.spyOn(service, 'createAnalyticsContext')
.mockReturnValue(mockContext);
const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId);
await context.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
expect(trackSpy).toHaveBeenCalledWith(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
});
it('should call pageview with correct parameters', async () => {
const pageviewSpy = jest.fn().mockResolvedValue({ success: true });
const mockContext = AnalyticsContextMock({
pageview: pageviewSpy,
});
jest
.spyOn(service, 'createAnalyticsContext')
.mockReturnValue(mockContext);
const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId);
const testPageviewProperties = {
href: '/test-url',
locale: '',
pathname: '',
referrer: '',
sessionId: '',
timeZone: '',
userAgent: '',
};
await context.pageview('page-view', testPageviewProperties);
expect(pageviewSpy).toHaveBeenCalledWith(
'page-view',
testPageviewProperties,
);
});
it('should return success when track is called', async () => {
const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId);
const result = await context.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
expect(result).toEqual({ success: true });
});
it('should return success when pageview is called', async () => {
const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId);
const result = await context.pageview('page-view', {});
expect(result).toEqual({ success: true });
});
});
});

View File

@ -0,0 +1,70 @@
import { Injectable } from '@nestjs/common';
import {
makePageview,
makeTrackEvent,
} from 'src/engine/core-modules/analytics/utils/analytics.utils';
import { ClickhouseService } from 'src/engine/core-modules/analytics/services/clickhouse.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import {
TrackEventName,
TrackEventProperties,
} from 'src/engine/core-modules/analytics/types/events.type';
import { PageviewProperties } from 'src/engine/core-modules/analytics/utils/events/pageview/pageview';
import {
AnalyticsException,
AnalyticsExceptionCode,
} from 'src/engine/core-modules/analytics/analytics.exception';
@Injectable()
export class AnalyticsService {
constructor(
private readonly twentyConfigService: TwentyConfigService,
private readonly clickhouseService: ClickhouseService,
) {}
createAnalyticsContext(context?: {
workspaceId?: string | null | undefined;
userId?: string | null | undefined;
}) {
const userIdAndWorkspaceId = context
? {
...(context.userId ? { userId: context.userId } : {}),
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
}
: {};
return {
track: <T extends TrackEventName>(
event: T,
properties: TrackEventProperties<T>,
) =>
this.preventAnalyticsIfDisabled(() =>
this.clickhouseService.pushEvent({
...userIdAndWorkspaceId,
...makeTrackEvent(event, properties),
}),
),
pageview: (name: string, properties: Partial<PageviewProperties>) =>
this.preventAnalyticsIfDisabled(() =>
this.clickhouseService.pushEvent({
...userIdAndWorkspaceId,
...makePageview(name, properties),
}),
),
};
}
private preventAnalyticsIfDisabled(
sendEventOrPageviewFunction: () => Promise<{ success: boolean }>,
) {
if (!this.twentyConfigService.get('ANALYTICS_ENABLED')) {
return { success: true };
}
try {
return sendEventOrPageviewFunction();
} catch (err) {
return new AnalyticsException(err, AnalyticsExceptionCode.INVALID_INPUT);
}
}
}

View File

@ -0,0 +1,159 @@
import { Test, TestingModule } from '@nestjs/testing';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated';
import {
makePageview,
makeTrackEvent,
} from 'src/engine/core-modules/analytics/utils/analytics.utils';
import { ClickhouseService } from './clickhouse.service';
// Mock the createClient function from @clickhouse/client
jest.mock('@clickhouse/client', () => ({
createClient: jest.fn().mockReturnValue({
insert: jest.fn().mockResolvedValue({}),
}),
}));
describe('ClickhouseService', () => {
let service: ClickhouseService;
let twentyConfigService: TwentyConfigService;
let exceptionHandlerService: ExceptionHandlerService;
let mockClickhouseClient: { insert: jest.Mock };
const mockPageview = makePageview('Home', {
href: 'https://example.com/test',
locale: 'en-US',
pathname: '/test',
referrer: 'https://example.com',
sessionId: 'test-session-id',
timeZone: 'UTC',
userAgent: 'test-user-agent',
});
const mockEvent = makeTrackEvent(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
beforeEach(async () => {
jest.clearAllMocks();
mockClickhouseClient = {
insert: jest.fn().mockResolvedValue({}),
};
const module: TestingModule = await Test.createTestingModule({
providers: [
ClickhouseService,
{
provide: TwentyConfigService,
useValue: {
get: jest.fn((key) => {
if (key === 'ANALYTICS_ENABLED') return true;
if (key === 'CLICKHOUSE_URL') return 'http://localhost:8123';
return null;
}),
},
},
{
provide: ExceptionHandlerService,
useValue: {
captureExceptions: jest.fn(),
},
},
],
}).compile();
service = module.get<ClickhouseService>(ClickhouseService);
twentyConfigService = module.get<TwentyConfigService>(TwentyConfigService);
exceptionHandlerService = module.get<ExceptionHandlerService>(
ExceptionHandlerService,
);
// Set the mock client
// @ts-expect-error accessing private property for testing
service.clickhouseClient = mockClickhouseClient;
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('constructor', () => {
it('should not initialize clickhouse client when analytics is disabled', async () => {
jest.spyOn(twentyConfigService, 'get').mockImplementation((key) => {
if (key === 'ANALYTICS_ENABLED') return false;
});
const newModule: TestingModule = await Test.createTestingModule({
providers: [
ClickhouseService,
{
provide: TwentyConfigService,
useValue: twentyConfigService,
},
{
provide: ExceptionHandlerService,
useValue: exceptionHandlerService,
},
],
}).compile();
const newService = newModule.get<ClickhouseService>(ClickhouseService);
// @ts-expect-error accessing private property for testing
expect(newService.clickhouseClient).toBeUndefined();
});
});
describe('pushEvent', () => {
it('should insert event into clickhouse and return success', async () => {
const result = await service.pushEvent(mockEvent);
expect(result).toEqual({ success: true });
const { type: _type, ...rest } = mockEvent;
expect(mockClickhouseClient.insert).toHaveBeenCalledWith({
table: 'events',
values: [rest],
format: 'JSONEachRow',
});
});
it('should insert pageview into clickhouse and return success', async () => {
const result = await service.pushEvent(mockPageview);
expect(result).toEqual({ success: true });
const { type: _type, ...rest } = mockPageview;
expect(mockClickhouseClient.insert).toHaveBeenCalledWith({
table: 'pageview',
values: [rest],
format: 'JSONEachRow',
});
});
it('should return success when clickhouse client is not defined', async () => {
// @ts-expect-error accessing private property for testing
service.clickhouseClient = undefined;
const result = await service.pushEvent(mockEvent);
expect(result).toEqual({ success: true });
});
it('should handle errors and return failure', async () => {
const testError = new Error('Test error');
mockClickhouseClient.insert.mockRejectedValueOnce(testError);
const result = await service.pushEvent(mockEvent);
expect(result).toEqual({ success: false });
expect(exceptionHandlerService.captureExceptions).toHaveBeenCalledWith([
testError,
]);
});
});
});

View File

@ -0,0 +1,60 @@
import { Injectable } from '@nestjs/common';
import { ClickHouseClient, createClient } from '@clickhouse/client';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import {
makePageview,
makeTrackEvent,
} from 'src/engine/core-modules/analytics/utils/analytics.utils';
@Injectable()
export class ClickhouseService {
private clickhouseClient: ClickHouseClient | undefined;
constructor(
private readonly exceptionHandlerService: ExceptionHandlerService,
private readonly twentyConfigService: TwentyConfigService,
) {
if (twentyConfigService.get('ANALYTICS_ENABLED')) {
this.clickhouseClient = createClient({
url: twentyConfigService.get('CLICKHOUSE_URL'),
compression: {
response: true,
request: true,
},
clickhouse_settings: {
async_insert: 1,
wait_for_async_insert: 1,
},
});
}
}
async pushEvent(
data: (
| ReturnType<typeof makeTrackEvent>
| ReturnType<typeof makePageview>
) & { userId?: string | null; workspaceId?: string | null },
) {
try {
if (!this.clickhouseClient) {
return { success: true };
}
const { type, ...rest } = data;
await this.clickhouseClient.insert({
table: type === 'page' ? 'pageview' : 'events',
values: [rest],
format: 'JSONEachRow',
});
return { success: true };
} catch (err) {
this.exceptionHandlerService.captureExceptions([err]);
return { success: false };
}
}
}

View File

@ -0,0 +1,2 @@
export type AnalyticsCommonPropertiesType = 'timestamp' | 'version';
export type IdentifierType = 'workspaceId' | 'userId';

View File

@ -0,0 +1,72 @@
import {
WEBHOOK_RESPONSE_EVENT,
WebhookResponseTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/webhook/webhook-response';
import {
SERVERLESS_FUNCTION_EXECUTED_EVENT,
ServerlessFunctionExecutedTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/serverless-function/serverless-function-executed';
import {
CUSTOM_DOMAIN_DEACTIVATED_EVENT,
CustomDomainDeactivatedTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated';
import {
CUSTOM_DOMAIN_ACTIVATED_EVENT,
CustomDomainActivatedTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated';
import {
WORKSPACE_ENTITY_CREATED_EVENT,
WorkspaceEntityCreatedTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/workspace-entity/workspace-entity-created';
import {
USER_SIGNUP_EVENT,
UserSignupTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/user/user-signup';
import {
MONITORING_EVENT,
MonitoringTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/monitoring/monitoring';
import {
OBJECT_RECORD_CREATED_EVENT,
ObjectRecordCreatedTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created';
import {
OBJECT_RECORD_UPDATED_EVENT,
ObjectRecordUpdatedTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated';
import {
OBJECT_RECORD_DELETED_EVENT,
ObjectRecordDeletedTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete';
// Define all track event names
export type TrackEventName =
| typeof CUSTOM_DOMAIN_ACTIVATED_EVENT
| typeof CUSTOM_DOMAIN_DEACTIVATED_EVENT
| typeof SERVERLESS_FUNCTION_EXECUTED_EVENT
| typeof WEBHOOK_RESPONSE_EVENT
| typeof WORKSPACE_ENTITY_CREATED_EVENT
| typeof MONITORING_EVENT
| typeof OBJECT_RECORD_CREATED_EVENT
| typeof OBJECT_RECORD_UPDATED_EVENT
| typeof OBJECT_RECORD_DELETED_EVENT
| typeof USER_SIGNUP_EVENT;
// Map event names to their corresponding event types
export interface TrackEvents {
[CUSTOM_DOMAIN_ACTIVATED_EVENT]: CustomDomainActivatedTrackEvent;
[CUSTOM_DOMAIN_DEACTIVATED_EVENT]: CustomDomainDeactivatedTrackEvent;
[SERVERLESS_FUNCTION_EXECUTED_EVENT]: ServerlessFunctionExecutedTrackEvent;
[WEBHOOK_RESPONSE_EVENT]: WebhookResponseTrackEvent;
[WORKSPACE_ENTITY_CREATED_EVENT]: WorkspaceEntityCreatedTrackEvent;
[USER_SIGNUP_EVENT]: UserSignupTrackEvent;
[MONITORING_EVENT]: MonitoringTrackEvent;
[OBJECT_RECORD_DELETED_EVENT]: ObjectRecordDeletedTrackEvent;
[OBJECT_RECORD_CREATED_EVENT]: ObjectRecordCreatedTrackEvent;
[OBJECT_RECORD_UPDATED_EVENT]: ObjectRecordUpdatedTrackEvent;
}
export type TrackEventProperties<T extends TrackEventName> =
T extends keyof TrackEvents
? TrackEvents[T]['properties']
: Record<string, unknown>;

View File

@ -0,0 +1,50 @@
import { format } from 'date-fns';
import { AnalyticsCommonPropertiesType } from 'src/engine/core-modules/analytics/types/common.type';
import {
PageviewProperties,
pageviewSchema,
} from 'src/engine/core-modules/analytics/utils/events/pageview/pageview';
import {
TrackEventName,
TrackEventProperties,
} from 'src/engine/core-modules/analytics/types/events.type';
import {
eventsRegistry,
GenericTrackEvent,
} from 'src/engine/core-modules/analytics/utils/events/track/track';
const common = (): Record<AnalyticsCommonPropertiesType, string> => ({
timestamp: format(new Date(), 'yyyy-MM-dd HH:mm:ss'),
version: '1',
});
export function makePageview(
name: string,
properties: Partial<PageviewProperties> = {},
) {
return pageviewSchema.parse({
type: 'page',
name,
...common(),
properties,
});
}
export function makeTrackEvent<T extends TrackEventName>(
event: T,
properties: TrackEventProperties<T>,
): GenericTrackEvent<T> {
const schema = eventsRegistry.get(event);
if (!schema) {
throw new Error(`Schema for event ${event} is not implemented`);
}
return schema.parse({
type: 'track',
event,
properties,
...common(),
});
}

View File

@ -0,0 +1,10 @@
import { z } from 'zod';
export const baseEventSchema = z
.object({
timestamp: z.string(),
userId: z.string().nullish(),
workspaceId: z.string().nullish(),
version: z.string(),
})
.strict();

View File

@ -0,0 +1,19 @@
import { z } from 'zod';
import { baseEventSchema } from 'src/engine/core-modules/analytics/utils/events/common/base-schemas';
export const pageviewSchema = baseEventSchema.extend({
type: z.literal('page'),
name: z.string(),
properties: z.object({
href: z.string(),
locale: z.string(),
pathname: z.string(),
referrer: z.string(),
sessionId: z.string(),
timeZone: z.string(),
userAgent: z.string(),
}),
});
export type PageviewProperties = z.infer<typeof pageviewSchema>['properties'];

View File

@ -0,0 +1,17 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const CUSTOM_DOMAIN_ACTIVATED_EVENT = 'Custom Domain Activated' as const;
export const customDomainActivatedSchema = z
.object({
event: z.literal(CUSTOM_DOMAIN_ACTIVATED_EVENT),
properties: z.object({}).strict(),
})
.strict();
export type CustomDomainActivatedTrackEvent = z.infer<
typeof customDomainActivatedSchema
>;
registerEvent(CUSTOM_DOMAIN_ACTIVATED_EVENT, customDomainActivatedSchema);

View File

@ -0,0 +1,18 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const CUSTOM_DOMAIN_DEACTIVATED_EVENT =
'Custom Domain Deactivated' as const;
export const customDomainDeactivatedSchema = z
.object({
event: z.literal(CUSTOM_DOMAIN_DEACTIVATED_EVENT),
properties: z.object({}).strict(),
})
.strict();
export type CustomDomainDeactivatedTrackEvent = z.infer<
typeof customDomainDeactivatedSchema
>;
registerEvent(CUSTOM_DOMAIN_DEACTIVATED_EVENT, customDomainDeactivatedSchema);

View File

@ -0,0 +1,22 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const MONITORING_EVENT = 'Monitoring' as const;
export const monitoringSchema = z
.object({
event: z.literal(MONITORING_EVENT),
properties: z
.object({
eventName: z.string(),
connectedAccountId: z.string().optional(),
messageChannelId: z.string().optional(),
message: z.string().optional(),
})
.strict(),
})
.strict();
export type MonitoringTrackEvent = z.infer<typeof monitoringSchema>;
registerEvent(MONITORING_EVENT, monitoringSchema);

View File

@ -0,0 +1,15 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const OBJECT_RECORD_CREATED_EVENT = 'Object Record Created' as const;
export const objectRecordCreatedSchema = z.object({
event: z.literal(OBJECT_RECORD_CREATED_EVENT),
properties: z.object({}).passthrough(),
});
export type ObjectRecordCreatedTrackEvent = z.infer<
typeof objectRecordCreatedSchema
>;
registerEvent(OBJECT_RECORD_CREATED_EVENT, objectRecordCreatedSchema);

View File

@ -0,0 +1,15 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const OBJECT_RECORD_DELETED_EVENT = 'Object Record Deleted' as const;
export const objectRecordDeletedSchema = z.object({
event: z.literal(OBJECT_RECORD_DELETED_EVENT),
properties: z.object({}).passthrough(),
});
export type ObjectRecordDeletedTrackEvent = z.infer<
typeof objectRecordDeletedSchema
>;
registerEvent(OBJECT_RECORD_DELETED_EVENT, objectRecordDeletedSchema);

View File

@ -0,0 +1,15 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const OBJECT_RECORD_UPDATED_EVENT = 'Object Record Updated' as const;
export const objectRecordUpdatedSchema = z.object({
event: z.literal(OBJECT_RECORD_UPDATED_EVENT),
properties: z.object({}).passthrough(),
});
export type ObjectRecordUpdatedTrackEvent = z.infer<
typeof objectRecordUpdatedSchema
>;
registerEvent(OBJECT_RECORD_UPDATED_EVENT, objectRecordUpdatedSchema);

View File

@ -0,0 +1,29 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const SERVERLESS_FUNCTION_EXECUTED_EVENT =
'Serverless Function Executed' as const;
export const serverlessFunctionExecutedSchema = z
.object({
event: z.literal(SERVERLESS_FUNCTION_EXECUTED_EVENT),
properties: z
.object({
duration: z.number(),
status: z.enum(['IDLE', 'SUCCESS', 'ERROR']),
errorType: z.string().optional(),
functionId: z.string(),
functionName: z.string(),
})
.strict(),
})
.strict();
export type ServerlessFunctionExecutedTrackEvent = z.infer<
typeof serverlessFunctionExecutedSchema
>;
registerEvent(
SERVERLESS_FUNCTION_EXECUTED_EVENT,
serverlessFunctionExecutedSchema,
);

View File

@ -0,0 +1,28 @@
import { z } from 'zod';
import { baseEventSchema } from 'src/engine/core-modules/analytics/utils/events/common/base-schemas';
export const genericTrackSchema = baseEventSchema.extend({
type: z.literal('track'),
event: z.string(),
properties: z.any(),
});
export type GenericTrackEvent<E extends string = string> = {
type: 'track';
event: E;
properties: any;
timestamp: string;
version: string;
userId?: string;
workspaceId?: string;
};
export const eventsRegistry = new Map<string, z.ZodSchema<any>>();
export function registerEvent<E extends string, S extends z.ZodObject<any>>(
event: E,
schema: S,
): void {
eventsRegistry.set(event, genericTrackSchema.merge(schema));
}

View File

@ -0,0 +1,15 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const USER_SIGNUP_EVENT = 'User Signup' as const;
export const userSignupSchema = z
.object({
event: z.literal(USER_SIGNUP_EVENT),
properties: z.object({}).strict(),
})
.strict();
export type UserSignupTrackEvent = z.infer<typeof userSignupSchema>;
registerEvent(USER_SIGNUP_EVENT, userSignupSchema);

View File

@ -0,0 +1,23 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const WEBHOOK_RESPONSE_EVENT = 'Webhook Response' as const;
export const webhookResponseSchema = z
.object({
event: z.literal(WEBHOOK_RESPONSE_EVENT),
properties: z
.object({
status: z.number().optional(),
success: z.boolean(),
url: z.string(),
webhookId: z.string(),
eventName: z.string(),
})
.strict(),
})
.strict();
export type WebhookResponseTrackEvent = z.infer<typeof webhookResponseSchema>;
registerEvent(WEBHOOK_RESPONSE_EVENT, webhookResponseSchema);

View File

@ -0,0 +1,20 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
export const WORKSPACE_ENTITY_CREATED_EVENT =
'Workspace Entity Created' as const;
export const workspaceEntityCreatedSchema = z
.object({
event: z.literal(WORKSPACE_ENTITY_CREATED_EVENT),
properties: z.object({
name: z.string(),
}),
})
.strict();
export type WorkspaceEntityCreatedTrackEvent = z.infer<
typeof workspaceEntityCreatedSchema
>;
registerEvent(WORKSPACE_ENTITY_CREATED_EVENT, workspaceEntityCreatedSchema);

View File

@ -0,0 +1,54 @@
import { GenericTrackEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated';
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete';
export const fixtures: Array<GenericTrackEvent> = [
{
type: 'track',
event: CUSTOM_DOMAIN_ACTIVATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
{
type: 'track',
event: CUSTOM_DOMAIN_DEACTIVATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
{
type: 'track',
event: OBJECT_RECORD_CREATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
{
type: 'track',
event: OBJECT_RECORD_UPDATED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
{
type: 'track',
event: OBJECT_RECORD_DELETED_EVENT,
timestamp: '2024-10-24T15:55:35.177',
version: '1',
userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
properties: {},
},
];

View File

@ -24,6 +24,8 @@ import { handleException } from 'src/engine/core-modules/exception-handler/http-
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { CloudflareSecretMatchGuard } from 'src/engine/core-modules/domain-manager/guards/cloudflare-secret.guard';
import { CustomDomainService } from 'src/engine/core-modules/domain-manager/services/custom-domain.service';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated';
@Controller('cloudflare')
@UseFilters(AuthRestApiExceptionFilter)
@ -34,6 +36,7 @@ export class CloudflareController {
private readonly domainManagerService: DomainManagerService,
private readonly customDomainService: CustomDomainService,
private readonly exceptionHandlerService: ExceptionHandlerService,
private readonly analyticsService: AnalyticsService,
) {}
@Post('custom-hostname-webhooks')
@ -57,6 +60,10 @@ export class CloudflareController {
if (!workspace) return;
const analytics = this.analyticsService.createAnalyticsContext({
workspaceId: workspace.id,
});
const customDomainDetails =
await this.customDomainService.getCustomDomainDetails(
req.body.data.data.hostname,
@ -83,6 +90,8 @@ export class CloudflareController {
...workspace,
...workspaceUpdated,
});
await analytics.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
}
return res.status(200).send();

View File

@ -3,6 +3,7 @@ import { getRepositoryToken } from '@nestjs/typeorm';
import { Request, Response } from 'express';
import { Repository } from 'typeorm';
import { AnalyticsContextMock } from 'test/utils/analytics-context.mock';
import { CloudflareController } from 'src/engine/core-modules/domain-manager/controllers/cloudflare.controller';
import { CustomDomainValidRecords } from 'src/engine/core-modules/domain-manager/dtos/custom-domain-valid-records';
@ -12,6 +13,7 @@ import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handl
import { HttpExceptionHandlerService } from 'src/engine/core-modules/exception-handler/http-exception-handler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
describe('CloudflareController - customHostnameWebhooks', () => {
let controller: CloudflareController;
@ -61,6 +63,12 @@ describe('CloudflareController - customHostnameWebhooks', () => {
get: jest.fn(),
},
},
{
provide: AnalyticsService,
useValue: {
createAnalyticsContext: AnalyticsContextMock,
},
},
],
}).compile();

View File

@ -5,9 +5,10 @@ import { DomainManagerService } from 'src/engine/core-modules/domain-manager/ser
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { CloudflareController } from 'src/engine/core-modules/domain-manager/controllers/cloudflare.controller';
import { CustomDomainService } from 'src/engine/core-modules/domain-manager/services/custom-domain.service';
import { AnalyticsModule } from 'src/engine/core-modules/analytics/analytics.module';
@Module({
imports: [TypeOrmModule.forFeature([Workspace], 'core')],
imports: [AnalyticsModule, TypeOrmModule.forFeature([Workspace], 'core')],
providers: [DomainManagerService, CustomDomainService],
exports: [DomainManagerService, CustomDomainService],
controllers: [CloudflareController],

View File

@ -2,10 +2,12 @@ import { Test, TestingModule } from '@nestjs/testing';
import Cloudflare from 'cloudflare';
import { CustomHostnameCreateResponse } from 'cloudflare/resources/custom-hostnames/custom-hostnames';
import { AnalyticsContextMock } from 'test/utils/analytics-context.mock';
import { DomainManagerException } from 'src/engine/core-modules/domain-manager/domain-manager.exception';
import { CustomDomainService } from 'src/engine/core-modules/domain-manager/services/custom-domain.service';
import { DomainManagerService } from 'src/engine/core-modules/domain-manager/services/domain-manager.service';
import { DomainManagerException } from 'src/engine/core-modules/domain-manager/domain-manager.exception';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
jest.mock('cloudflare');
@ -25,6 +27,12 @@ describe('CustomDomainService', () => {
get: jest.fn(),
},
},
{
provide: AnalyticsService,
useValue: {
createAnalyticsContext: AnalyticsContextMock,
},
},
{
provide: DomainManagerService,
useValue: {

View File

@ -463,6 +463,18 @@ export class ConfigVariables {
@IsBoolean()
ANALYTICS_ENABLED = false;
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.AnalyticsConfig,
description: 'Clickhouse host for analytics',
})
@IsOptional()
@IsUrl({
require_tld: false,
allow_underscores: true,
})
@ValidateIf((env) => env.ANALYTICS_ENABLED === true)
CLICKHOUSE_URL: string;
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.Logging,
description: 'Enable or disable telemetry logging',

View File

@ -5,7 +5,6 @@ import { Repository } from 'typeorm';
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action';
import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants';
import { AuthException } from 'src/engine/core-modules/auth/auth.exception';
import { DomainManagerService } from 'src/engine/core-modules/domain-manager/services/domain-manager.service';
import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity';
@ -21,6 +20,7 @@ import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.
import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants';
describe('UserWorkspaceService', () => {
let service: UserWorkspaceService;

View File

@ -20,7 +20,7 @@ import { In, Repository } from 'typeorm';
import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface';
import { SupportDriver } from 'src/engine/core-modules/twenty-config/interfaces/support.interface';
import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import {
AuthException,
AuthExceptionCode,

View File

@ -25,6 +25,7 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service';
import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service';
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
describe('WorkspaceService', () => {
let service: WorkspaceService;
@ -74,8 +75,13 @@ describe('WorkspaceService', () => {
deleteSubscriptions: jest.fn(),
},
},
{
provide: AnalyticsService,
useValue: {
createAnalyticsContext: jest.fn(),
},
},
...[
WorkspaceManagerService,
WorkspaceManagerService,
UserWorkspaceService,
UserService,

View File

@ -44,6 +44,9 @@ import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage
import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service';
import { DEFAULT_FEATURE_FLAGS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags';
import { extractVersionMajorMinorPatch } from 'src/utils/version/extract-version-major-minor-patch';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated';
@Injectable()
// eslint-disable-next-line @nx/workspace-inject-workspace-repository
@ -67,6 +70,7 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
private readonly domainManagerService: DomainManagerService,
private readonly exceptionHandlerService: ExceptionHandlerService,
private readonly permissionsService: PermissionsService,
private readonly analyticsService: AnalyticsService,
private readonly customDomainService: CustomDomainService,
private readonly workspaceCacheStorageService: WorkspaceCacheStorageService,
@InjectMessageQueue(MessageQueue.deleteCascadeQueue)
@ -413,6 +417,17 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
if (workspace.isCustomDomainEnabled !== isCustomDomainWorking) {
workspace.isCustomDomainEnabled = isCustomDomainWorking;
await this.workspaceRepository.save(workspace);
const analytics = this.analyticsService.createAnalyticsContext({
workspaceId: workspace.id,
});
analytics.track(
workspace.isCustomDomainEnabled
? CUSTOM_DOMAIN_ACTIVATED_EVENT
: CUSTOM_DOMAIN_DEACTIVATED_EVENT,
{},
);
}
return customDomainDetails;

View File

@ -24,6 +24,7 @@ import { RoleModule } from 'src/engine/metadata-modules/role/role.module';
import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module';
import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module';
import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module';
import { AnalyticsModule } from 'src/engine/core-modules/analytics/analytics.module';
import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts';
import { Workspace } from './workspace.entity';
@ -54,6 +55,7 @@ import { WorkspaceService } from './services/workspace.service';
TypeORMModule,
PermissionsModule,
WorkspaceCacheStorageModule,
AnalyticsModule,
RoleModule,
],
services: [WorkspaceService],

View File

@ -10,7 +10,7 @@ import { IsNull, Not, Repository } from 'typeorm';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
@ -35,6 +35,7 @@ import {
ServerlessFunctionException,
ServerlessFunctionExceptionCode,
} from 'src/engine/metadata-modules/serverless-function/serverless-function.exception';
import { SERVERLESS_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/serverless-function/serverless-function-executed';
@Injectable()
export class ServerlessFunctionService {
@ -143,9 +144,11 @@ export class ServerlessFunctionService {
version,
);
const eventInput = {
action: 'serverlessFunction.executed',
payload: {
this.analyticsService
.createAnalyticsContext({
workspaceId,
})
.track(SERVERLESS_FUNCTION_EXECUTED_EVENT, {
duration: resultServerlessFunction.duration,
status: resultServerlessFunction.status,
...(resultServerlessFunction.error && {
@ -153,14 +156,7 @@ export class ServerlessFunctionService {
}),
functionId: functionToExecute.id,
functionName: functionToExecute.name,
},
};
this.analyticsService.create(
eventInput,
'serverless-function',
workspaceId,
);
});
return resultServerlessFunction;
}

View File

@ -1,7 +1,7 @@
import { Injectable } from '@nestjs/common';
import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { MONITORING_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/monitoring/monitoring';
type MessagingTelemetryTrackInput = {
eventName: string;
@ -14,10 +14,7 @@ type MessagingTelemetryTrackInput = {
@Injectable()
export class MessagingTelemetryService {
constructor(
private readonly analyticsService: AnalyticsService,
private readonly twentyConfigService: TwentyConfigService,
) {}
constructor(private readonly analyticsService: AnalyticsService) {}
public async track({
eventName,
@ -27,20 +24,16 @@ export class MessagingTelemetryService {
messageChannelId,
message,
}: MessagingTelemetryTrackInput): Promise<void> {
await this.analyticsService.create(
{
action: 'monitoring',
payload: {
eventName: `messaging.${eventName}`,
workspaceId,
userId,
connectedAccountId,
messageChannelId,
message,
},
},
userId,
workspaceId,
);
await this.analyticsService
.createAnalyticsContext({
userId,
workspaceId,
})
.track(MONITORING_EVENT, {
eventName: `messaging.${eventName}`,
connectedAccountId,
messageChannelId,
message,
});
}
}

View File

@ -8,6 +8,10 @@ import { AuditLogRepository } from 'src/modules/timeline/repositiories/audit-log
import { AuditLogWorkspaceEntity } from 'src/modules/timeline/standard-objects/audit-log.workspace-entity';
import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created';
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete';
@Processor(MessageQueue.entityEventsToDbQueue)
export class CreateAuditLogFromInternalEvent {
@ -16,6 +20,7 @@ export class CreateAuditLogFromInternalEvent {
private readonly workspaceMemberService: WorkspaceMemberRepository,
@InjectObjectMetadataRepository(AuditLogWorkspaceEntity)
private readonly auditLogRepository: AuditLogRepository,
private readonly analyticsService: AnalyticsService,
) {}
@Process(CreateAuditLogFromInternalEvent.name)
@ -48,6 +53,19 @@ export class CreateAuditLogFromInternalEvent {
eventData.recordId,
workspaceEventBatch.workspaceId,
);
const analytics = this.analyticsService.createAnalyticsContext({
workspaceId: workspaceEventBatch.workspaceId,
userId: eventData.userId,
});
if (workspaceEventBatch.name.endsWith('.updated')) {
analytics.track(OBJECT_RECORD_UPDATED_EVENT, eventData.properties);
} else if (workspaceEventBatch.name.endsWith('.created')) {
analytics.track(OBJECT_RECORD_CREATED_EVENT, eventData.properties);
} else if (workspaceEventBatch.name.endsWith('.deleted')) {
analytics.track(OBJECT_RECORD_DELETED_EVENT, eventData.properties);
}
}
}
}

View File

@ -6,6 +6,7 @@ import { UpsertTimelineActivityFromInternalEvent } from 'src/modules/timeline/jo
import { AuditLogWorkspaceEntity } from 'src/modules/timeline/standard-objects/audit-log.workspace-entity';
import { TimelineActivityModule } from 'src/modules/timeline/timeline-activity.module';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
import { AnalyticsModule } from 'src/engine/core-modules/analytics/analytics.module';
@Module({
imports: [
@ -14,6 +15,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta
AuditLogWorkspaceEntity,
]),
TimelineActivityModule,
AnalyticsModule,
],
providers: [
CreateAuditLogFromInternalEvent,

View File

@ -3,10 +3,11 @@ import { Logger } from '@nestjs/common';
import crypto from 'crypto';
import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service';
import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/webhook/webhook-response';
export type CallWebhookJobData = {
targetUrl: string;
@ -46,6 +47,9 @@ export class CallWebhookJob {
webhookId: data.webhookId,
eventName: data.eventName,
};
const analytics = this.analyticsService.createAnalyticsContext({
workspaceId: data.workspaceId,
});
try {
const headers: Record<string, string> = {
@ -73,27 +77,18 @@ export class CallWebhookJob {
);
const success = response.status >= 200 && response.status < 300;
const eventInput = {
action: 'webhook.response',
payload: {
status: response.status,
success,
...commonPayload,
},
};
this.analyticsService.create(eventInput, 'webhook', data.workspaceId);
analytics.track(WEBHOOK_RESPONSE_EVENT, {
status: response.status,
success,
...commonPayload,
});
} catch (err) {
const eventInput = {
action: 'webhook.response',
payload: {
success: false,
...commonPayload,
...(err.response && { status: err.response.status }),
},
};
this.analyticsService.create(eventInput, 'webhook', data.workspaceId);
analytics.track(WEBHOOK_RESPONSE_EVENT, {
success: false,
...commonPayload,
...(err.response && { status: err.response.status }),
});
this.logger.error(
`Error calling webhook on targetUrl '${data.targetUrl}': ${err}`,
);

View File

@ -0,0 +1,74 @@
import process from 'process';
import request from 'supertest';
import { createClient, ClickHouseClient } from '@clickhouse/client';
import { GenericTrackEvent } from 'src/engine/core-modules/analytics/utils/events/track/track';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created';
describe('ClickHouse Event Registration (integration)', () => {
let clickhouseClient: ClickHouseClient;
beforeAll(async () => {
jest.useRealTimers();
clickhouseClient = createClient({
url: process.env.CLICKHOUSE_URL,
});
await clickhouseClient.query({
query: 'TRUNCATE TABLE events',
format: 'JSONEachRow',
});
});
afterAll(async () => {
if (clickhouseClient) {
await clickhouseClient.close();
}
});
it('should register events in ClickHouse when sending an event', async () => {
const mutation = `
mutation TrackAnalytics($type: AnalyticsType!, $event: String, $name: String, $properties: JSON) {
trackAnalytics(type: $type, event: $event, name: $name, properties: $properties) {
success
}
}
`;
const variables = {
type: 'TRACK',
event: OBJECT_RECORD_CREATED_EVENT,
properties: {},
};
const response = await request(`http://localhost:${APP_PORT}`)
.post('/graphql')
.send({
query: mutation,
variables,
});
expect(response.status).toBe(200);
expect(response.body.data.trackAnalytics.success).toBe(true);
const queryResult = await clickhouseClient.query({
query: `
SELECT *
FROM events
WHERE event = '${OBJECT_RECORD_CREATED_EVENT}' AND timestamp >= now() - INTERVAL 1 SECOND
`,
format: 'JSONEachRow',
});
const rows = await queryResult.json<GenericTrackEvent>();
expect(rows.length).toEqual(1);
expect(rows[0].properties).toEqual(variables.properties);
expect(rows[0].event).toEqual(variables.event);
expect(rows[0].workspaceId).toEqual('');
expect(rows[0].userId).toEqual('');
expect(rows[0].timestamp).toHaveLength(23);
});
});

View File

@ -0,0 +1,19 @@
import { TrackEventName } from 'src/engine/core-modules/analytics/types/events.type';
export const AnalyticsContextMock = (params?: {
track?:
| ((
event: TrackEventName,
properties: any,
) => Promise<{ success: boolean }>)
| jest.Mock<any, any>;
pageview?:
| ((name: string, properties: any) => Promise<{ success: boolean }>)
| jest.Mock<any, any>;
}) => {
return {
track: params?.track ?? jest.fn().mockResolvedValue({ success: true }),
pageview:
params?.pageview ?? jest.fn().mockResolvedValue({ success: true }),
};
};

View File

@ -5194,6 +5194,22 @@ __metadata:
languageName: node
linkType: hard
"@clickhouse/client-common@npm:1.11.0":
version: 1.11.0
resolution: "@clickhouse/client-common@npm:1.11.0"
checksum: 10c0/84a74458acdf279bc4e844c235d0f658cb77f82e90f19fa285481c31f2f17210a5e801a61f080d6dfd100d970bc4e15a5692d2b7fd067502fbaad99d0dc51310
languageName: node
linkType: hard
"@clickhouse/client@npm:^1.11.0":
version: 1.11.0
resolution: "@clickhouse/client@npm:1.11.0"
dependencies:
"@clickhouse/client-common": "npm:1.11.0"
checksum: 10c0/8557de33dd9c88d2813d563f89b49e666dc2d8166c9131e9727f6935db8dc65428927b4b4ab20b1bc7646b9c38c1eb8d1e372ba483b0b2bb69d64ed89488dec2
languageName: node
linkType: hard
"@codemirror/autocomplete@npm:^6.0.0, @codemirror/autocomplete@npm:^6.4.0":
version: 6.18.0
resolution: "@codemirror/autocomplete@npm:6.18.0"
@ -54989,6 +55005,7 @@ __metadata:
version: 0.0.0-use.local
resolution: "twenty-server@workspace:packages/twenty-server"
dependencies:
"@clickhouse/client": "npm:^1.11.0"
"@esbuild-plugins/node-modules-polyfill": "npm:^0.2.2"
"@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch"
"@langchain/mistralai": "npm:^0.0.24"