diff --git a/.github/workflows/ci-server.yaml b/.github/workflows/ci-server.yaml index 2cc724eb1..78fcae457 100644 --- a/.github/workflows/ci-server.yaml +++ b/.github/workflows/ci-server.yaml @@ -168,9 +168,25 @@ jobs: image: redis ports: - 6379:6379 + clickhouse: + image: clickhouse/clickhouse-server:latest + env: + CLICKHOUSE_PASSWORD: clickhousePassword + CLICKHOUSE_URL: "http://default:clickhousePassword@localhost:8123/twenty" + ports: + - 8123:8123 + - 9000:9000 + options: >- + --health-cmd "clickhouse-client --host=localhost --port=9000 --user=default --password=clickhousePassword --query='SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 env: NX_REJECT_UNKNOWN_LOCAL_CACHE: 0 NODE_ENV: test + ANALYTICS_ENABLED: true + CLICKHOUSE_URL: "http://default:clickhousePassword@localhost:8123/twenty" + CLICKHOUSE_PASSWORD: clickhousePassword steps: - name: Fetch custom Github Actions and base branch history uses: actions/checkout@v4 @@ -178,7 +194,7 @@ jobs: fetch-depth: 0 - name: Install dependencies uses: ./.github/workflows/actions/yarn-install - - name: Update .env.test for billing + - name: Update .env.test for integrations tests run: | echo "IS_BILLING_ENABLED=true" >> .env.test echo "BILLING_STRIPE_API_KEY=test-api-key" >> .env.test @@ -198,6 +214,10 @@ jobs: - name: Server / Create Test DB run: | PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d postgres -c 'CREATE DATABASE "test";' + - name: Run ClickHouse migrations + run: npx nx clickhouse:migrate twenty-server + - name: Run ClickHouse seeds + run: npx nx clickhouse:seed twenty-server - name: Server / Run Integration Tests uses: ./.github/workflows/actions/nx-affected with: diff --git a/Makefile b/Makefile index 851203d1a..f74579943 100644 --- a/Makefile +++ b/Makefile @@ -17,4 +17,7 @@ postgres-on-docker: -c "CREATE DATABASE \"test\" WITH OWNER postgres;" redis-on-docker: - docker run -d --name twenty_redis -p 6379:6379 redis/redis-stack-server:latest \ No newline at end of file + docker run -d --name twenty_redis -p 6379:6379 redis/redis-stack-server:latest + +clickhouse-on-docker: + docker run -d --name twenty_clickhouse -p 8123:8123 -p 9000:9000 -e CLICKHOUSE_PASSWORD=devPassword clickhouse/clickhouse-server:latest \ No newline at end of file diff --git a/packages/twenty-front/.eslintrc.cjs b/packages/twenty-front/.eslintrc.cjs index a8a5418d5..fc56645f7 100644 --- a/packages/twenty-front/.eslintrc.cjs +++ b/packages/twenty-front/.eslintrc.cjs @@ -1,5 +1,3 @@ -const path = require('path'); - module.exports = { extends: ['../../.eslintrc.react.cjs'], ignorePatterns: [ diff --git a/packages/twenty-front/src/generated-metadata/graphql.ts b/packages/twenty-front/src/generated-metadata/graphql.ts index 6c53b738e..3be34163c 100644 --- a/packages/twenty-front/src/generated-metadata/graphql.ts +++ b/packages/twenty-front/src/generated-metadata/graphql.ts @@ -63,6 +63,11 @@ export type Analytics = { success: Scalars['Boolean']['output']; }; +export enum AnalyticsType { + PAGEVIEW = 'PAGEVIEW', + TRACK = 'TRACK' +} + export type ApiConfig = { __typename?: 'ApiConfig'; mutationMaximumAffectedRecords: Scalars['Float']['output']; @@ -155,7 +160,8 @@ export type BillingEndTrialPeriodOutput = { export type BillingMeteredProductUsageOutput = { __typename?: 'BillingMeteredProductUsageOutput'; - includedFreeQuantity: Scalars['Float']['output']; + freeTierQuantity: Scalars['Float']['output']; + freeTrialQuantity: Scalars['Float']['output']; periodEnd: Scalars['DateTime']['output']; periodStart: Scalars['DateTime']['output']; productKey: BillingProductKey; @@ -474,6 +480,10 @@ export type CreateServerlessFunctionInput = { }; export type CreateWorkflowVersionStepInput = { + /** Next step ID */ + nextStepId?: InputMaybe; + /** Parent step ID */ + parentStepId?: InputMaybe; /** New step type */ stepType: Scalars['String']['input']; /** Workflow version ID */ @@ -598,6 +608,12 @@ export type FeatureFlag = { workspaceId: Scalars['String']['output']; }; +export type FeatureFlagDto = { + __typename?: 'FeatureFlagDTO'; + key: FeatureFlagKey; + value: Scalars['Boolean']['output']; +}; + export enum FeatureFlagKey { IsAirtableIntegrationEnabled = 'IsAirtableIntegrationEnabled', IsAnalyticsV2Enabled = 'IsAnalyticsV2Enabled', @@ -970,8 +986,9 @@ export type Mutation = { syncRemoteTable: RemoteTable; syncRemoteTableSchemaChanges: RemoteTable; track: Analytics; + trackAnalytics: Analytics; unsyncRemoteTable: RemoteTable; - updateLabPublicFeatureFlag: FeatureFlag; + updateLabPublicFeatureFlag: FeatureFlagDto; updateOneField: Field; updateOneObject: Object; updateOneRemoteServer: RemoteServer; @@ -1252,6 +1269,14 @@ export type MutationTrackArgs = { }; +export type MutationTrackAnalyticsArgs = { + event?: InputMaybe; + name?: InputMaybe; + properties?: InputMaybe; + type: AnalyticsType; +}; + + export type MutationUnsyncRemoteTableArgs = { input: RemoteTableInput; }; @@ -2462,7 +2487,7 @@ export type Workspace = { defaultRole?: Maybe; deletedAt?: Maybe; displayName?: Maybe; - featureFlags?: Maybe>; + featureFlags?: Maybe>; hasValidEnterpriseKey: Scalars['Boolean']['output']; id: Scalars['UUID']['output']; inviteHash?: Maybe; diff --git a/packages/twenty-front/src/generated/graphql.tsx b/packages/twenty-front/src/generated/graphql.tsx index 9a01771ac..6ebb41371 100644 --- a/packages/twenty-front/src/generated/graphql.tsx +++ b/packages/twenty-front/src/generated/graphql.tsx @@ -55,6 +55,11 @@ export type Analytics = { success: Scalars['Boolean']; }; +export enum AnalyticsType { + PAGEVIEW = 'PAGEVIEW', + TRACK = 'TRACK' +} + export type ApiConfig = { __typename?: 'ApiConfig'; mutationMaximumAffectedRecords: Scalars['Float']; @@ -899,6 +904,7 @@ export type Mutation = { submitFormStep: Scalars['Boolean']; switchToYearlyInterval: BillingUpdateOutput; track: Analytics; + trackAnalytics: Analytics; updateLabPublicFeatureFlag: FeatureFlagDto; updateOneField: Field; updateOneObject: Object; @@ -1139,6 +1145,14 @@ export type MutationTrackArgs = { }; +export type MutationTrackAnalyticsArgs = { + event?: InputMaybe; + name?: InputMaybe; + properties?: InputMaybe; + type: AnalyticsType; +}; + + export type MutationUpdateLabPublicFeatureFlagArgs = { input: UpdateLabPublicFeatureFlagInput; }; @@ -2403,6 +2417,16 @@ export type GetTimelineThreadsFromPersonIdQueryVariables = Exact<{ export type GetTimelineThreadsFromPersonIdQuery = { __typename?: 'Query', getTimelineThreadsFromPersonId: { __typename?: 'TimelineThreadsWithTotal', totalNumberOfThreads: number, timelineThreads: Array<{ __typename?: 'TimelineThread', id: any, read: boolean, visibility: MessageChannelVisibility, lastMessageReceivedAt: string, lastMessageBody: string, subject: string, numberOfMessagesInThread: number, participantCount: number, firstParticipant: { __typename?: 'TimelineThreadParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string }, lastTwoParticipants: Array<{ __typename?: 'TimelineThreadParticipant', personId?: any | null, workspaceMemberId?: any | null, firstName: string, lastName: string, displayName: string, avatarUrl: string, handle: string }> }> } }; +export type TrackAnalyticsMutationVariables = Exact<{ + type: AnalyticsType; + event?: InputMaybe; + name?: InputMaybe; + properties?: InputMaybe; +}>; + + +export type TrackAnalyticsMutation = { __typename?: 'Mutation', trackAnalytics: { __typename?: 'Analytics', success: boolean } }; + export type TrackMutationVariables = Exact<{ action: Scalars['String']; payload: Scalars['JSON']; @@ -3326,6 +3350,42 @@ export function useGetTimelineThreadsFromPersonIdLazyQuery(baseOptions?: Apollo. export type GetTimelineThreadsFromPersonIdQueryHookResult = ReturnType; export type GetTimelineThreadsFromPersonIdLazyQueryHookResult = ReturnType; export type GetTimelineThreadsFromPersonIdQueryResult = Apollo.QueryResult; +export const TrackAnalyticsDocument = gql` + mutation TrackAnalytics($type: AnalyticsType!, $event: String, $name: String, $properties: JSON) { + trackAnalytics(type: $type, event: $event, name: $name, properties: $properties) { + success + } +} + `; +export type TrackAnalyticsMutationFn = Apollo.MutationFunction; + +/** + * __useTrackAnalyticsMutation__ + * + * To run a mutation, you first call `useTrackAnalyticsMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useTrackAnalyticsMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [trackAnalyticsMutation, { data, loading, error }] = useTrackAnalyticsMutation({ + * variables: { + * type: // value for 'type' + * event: // value for 'event' + * name: // value for 'name' + * properties: // value for 'properties' + * }, + * }); + */ +export function useTrackAnalyticsMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(TrackAnalyticsDocument, options); + } +export type TrackAnalyticsMutationHookResult = ReturnType; +export type TrackAnalyticsMutationResult = Apollo.MutationResult; +export type TrackAnalyticsMutationOptions = Apollo.BaseMutationOptions; export const TrackDocument = gql` mutation Track($action: String!, $payload: JSON!) { track(action: $action, payload: $payload) { diff --git a/packages/twenty-front/src/modules/analytics/graphql/queries/track.ts b/packages/twenty-front/src/modules/analytics/graphql/queries/track.ts index 44828ddae..1a71a6989 100644 --- a/packages/twenty-front/src/modules/analytics/graphql/queries/track.ts +++ b/packages/twenty-front/src/modules/analytics/graphql/queries/track.ts @@ -1,8 +1,18 @@ import { gql } from '@apollo/client'; -export const TRACK = gql` - mutation Track($action: String!, $payload: JSON!) { - track(action: $action, payload: $payload) { +export const TRACK_ANALYTICS = gql` + mutation TrackAnalytics( + $type: AnalyticsType! + $event: String + $name: String + $properties: JSON + ) { + trackAnalytics( + type: $type + event: $event + name: $name + properties: $properties + ) { success } } diff --git a/packages/twenty-front/src/modules/analytics/hooks/__tests__/useEventTracker.test.tsx b/packages/twenty-front/src/modules/analytics/hooks/__tests__/useEventTracker.test.tsx index a49b85612..6266e27b7 100644 --- a/packages/twenty-front/src/modules/analytics/hooks/__tests__/useEventTracker.test.tsx +++ b/packages/twenty-front/src/modules/analytics/hooks/__tests__/useEventTracker.test.tsx @@ -5,23 +5,76 @@ import { act, renderHook, waitFor } from '@testing-library/react'; import { ReactNode } from 'react'; import { RecoilRoot } from 'recoil'; -import { useEventTracker } from '../useEventTracker'; +import { ANALYTICS_COOKIE_NAME, useEventTracker } from '../useEventTracker'; +import { AnalyticsType } from '~/generated/graphql'; + +// Mock document.cookie +Object.defineProperty(document, 'cookie', { + writable: true, + value: `${ANALYTICS_COOKIE_NAME}=exampleId`, +}); const mocks: MockedResponse[] = [ { request: { query: gql` - mutation Track($action: String!, $payload: JSON!) { - track(action: $action, payload: $payload) { + mutation TrackAnalytics( + $type: AnalyticsType! + $event: String + $name: String + $properties: JSON + ) { + trackAnalytics( + type: $type + event: $event + name: $name + properties: $properties + ) { success } } `, variables: { - action: 'exampleType', - payload: { + type: AnalyticsType['TRACK'], + event: 'Example Event', + properties: { + foo: 'bar', + }, + }, + }, + result: jest.fn(() => ({ + data: { + track: { + success: true, + }, + }, + })), + }, + { + request: { + query: gql` + mutation TrackAnalytics( + $type: AnalyticsType! + $event: String + $name: String + $properties: JSON + ) { + trackAnalytics( + type: $type + event: $event + name: $name + properties: $properties + ) { + success + } + } + `, + variables: { + type: AnalyticsType['PAGEVIEW'], + name: 'Example', + properties: { sessionId: 'exampleId', - pathname: '', + pathname: '/example/path', userAgent: '', timeZone: '', locale: '', @@ -50,24 +103,45 @@ const Wrapper = ({ children }: { children: ReactNode }) => ( describe('useEventTracker', () => { it('should make the call to track the event', async () => { - const eventType = 'exampleType'; - const eventData = { - sessionId: 'exampleId', - pathname: '', - userAgent: '', - timeZone: '', - locale: '', - href: '', - referrer: '', + const payload = { + event: 'Example Event', + properties: { + foo: 'bar', + }, }; + const { result } = renderHook(() => useEventTracker(), { wrapper: Wrapper, }); act(() => { - result.current(eventType, eventData); + result.current(AnalyticsType['TRACK'], payload); }); await waitFor(() => { expect(mocks[0].result).toHaveBeenCalled(); }); }); + + it('should make the call to track a pageview', async () => { + const payload = { + name: 'Example', + properties: { + sessionId: 'exampleId', + pathname: '/example/path', + userAgent: '', + timeZone: '', + locale: '', + href: '', + referrer: '', + }, + }; + const { result } = renderHook(() => useEventTracker(), { + wrapper: Wrapper, + }); + act(() => { + result.current(AnalyticsType['PAGEVIEW'], payload); + }); + await waitFor(() => { + expect(mocks[1].result).toHaveBeenCalled(); + }); + }); }); diff --git a/packages/twenty-front/src/modules/analytics/hooks/useEventTracker.ts b/packages/twenty-front/src/modules/analytics/hooks/useEventTracker.ts index 4e90eb16f..a12fa61ec 100644 --- a/packages/twenty-front/src/modules/analytics/hooks/useEventTracker.ts +++ b/packages/twenty-front/src/modules/analytics/hooks/useEventTracker.ts @@ -1,14 +1,11 @@ import { useCallback } from 'react'; import { v4 } from 'uuid'; -import { useTrackMutation } from '~/generated/graphql'; -export interface EventData { - pathname: string; - userAgent: string; - timeZone: string; - locale: string; - href: string; - referrer: string; -} +import { + AnalyticsType, + MutationTrackAnalyticsArgs, + useTrackAnalyticsMutation, +} from '~/generated/graphql'; + export const ANALYTICS_COOKIE_NAME = 'analyticsCookie'; export const getSessionId = (): string => { const cookie: { [key: string]: string } = {}; @@ -28,16 +25,22 @@ export const setSessionId = (domain?: string): void => { }; export const useEventTracker = () => { - const [createEventMutation] = useTrackMutation(); + const [createEventMutation] = useTrackAnalyticsMutation(); return useCallback( - (eventAction: string, eventPayload: EventData) => { + ( + type: AnalyticsType, + payload: Omit, + ) => { createEventMutation({ variables: { - action: eventAction, - payload: { - sessionId: getSessionId(), - ...eventPayload, + type, + ...payload, + properties: { + ...payload.properties, + ...(type === AnalyticsType['PAGEVIEW'] + ? { sessionId: getSessionId() } + : {}), }, }, }); diff --git a/packages/twenty-front/src/modules/app/effect-components/PageChangeEffect.tsx b/packages/twenty-front/src/modules/app/effect-components/PageChangeEffect.tsx index 01ccee726..775a4b4f7 100644 --- a/packages/twenty-front/src/modules/app/effect-components/PageChangeEffect.tsx +++ b/packages/twenty-front/src/modules/app/effect-components/PageChangeEffect.tsx @@ -27,6 +27,8 @@ import { isDefined } from 'twenty-shared/utils'; import { useIsMatchingLocation } from '~/hooks/useIsMatchingLocation'; import { usePageChangeEffectNavigateLocation } from '~/hooks/usePageChangeEffectNavigateLocation'; import { useInitializeQueryParamState } from '~/modules/app/hooks/useInitializeQueryParamState'; +import { AnalyticsType } from '~/generated/graphql'; +import { getPageTitleFromPath } from '~/utils/title-utils'; // TODO: break down into smaller functions and / or hooks // - moved usePageChangeEffectNavigateLocation into dedicated hook @@ -174,13 +176,16 @@ export const PageChangeEffect = () => { useEffect(() => { setTimeout(() => { setSessionId(); - eventTracker('pageview', { - pathname: location.pathname, - locale: navigator.language, - userAgent: window.navigator.userAgent, - href: window.location.href, - referrer: document.referrer, - timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + eventTracker(AnalyticsType['PAGEVIEW'], { + name: getPageTitleFromPath(location.pathname), + properties: { + pathname: location.pathname, + locale: navigator.language, + userAgent: window.navigator.userAgent, + href: window.location.href, + referrer: document.referrer, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, }); }, 500); }, [eventTracker, location.pathname]); diff --git a/packages/twenty-front/src/testing/graphqlMocks.ts b/packages/twenty-front/src/testing/graphqlMocks.ts index a138b5307..670c85e71 100644 --- a/packages/twenty-front/src/testing/graphqlMocks.ts +++ b/packages/twenty-front/src/testing/graphqlMocks.ts @@ -1,7 +1,7 @@ import { getOperationName } from '@apollo/client/utilities'; import { graphql, GraphQLQuery, http, HttpResponse } from 'msw'; -import { TRACK } from '@/analytics/graphql/queries/track'; +import { TRACK_ANALYTICS } from '@/analytics/graphql/queries/track'; import { GET_CLIENT_CONFIG } from '@/client-config/graphql/queries/getClientConfig'; import { FIND_MANY_OBJECT_METADATA_ITEMS } from '@/object-metadata/graphql/queries'; import { GET_CURRENT_USER } from '@/users/graphql/queries/getCurrentUser'; @@ -110,10 +110,10 @@ export const graphqlMocks = { }); }, ), - graphql.mutation(getOperationName(TRACK) ?? '', () => { + graphql.mutation(getOperationName(TRACK_ANALYTICS) ?? '', () => { return HttpResponse.json({ data: { - track: { success: 1, __typename: 'TRACK' }, + track: { success: 1, __typename: 'TRACK_ANALYTICS' }, }, }); }), diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index e81da3a79..a62ebf985 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -77,4 +77,5 @@ FRONTEND_URL=http://localhost:3001 # CLOUDFLARE_ZONE_ID= # CLOUDFLARE_WEBHOOK_SECRET= # IS_CONFIG_VARIABLES_IN_DB_ENABLED=false - +# ANALYTICS_ENABLED= +# CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty \ No newline at end of file diff --git a/packages/twenty-server/.env.test b/packages/twenty-server/.env.test index c5f81ad3e..42648eb28 100644 --- a/packages/twenty-server/.env.test +++ b/packages/twenty-server/.env.test @@ -21,3 +21,5 @@ MESSAGING_PROVIDER_GMAIL_CALLBACK_URL=http://localhost:3000/auth/google-gmail/ge AUTH_MICROSOFT_CALLBACK_URL=http://localhost:3000/auth/microsoft/redirect AUTH_MICROSOFT_APIS_CALLBACK_URL=http://localhost:3000/auth/microsoft-apis/get-access-token + +CLICKHOUSE_URL=http://default:clickhousePassword@localhost:8123/twenty diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index ed49a3246..365bd7c8f 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -15,6 +15,7 @@ "typeorm": "../../node_modules/typeorm/.bin/typeorm" }, "dependencies": { + "@clickhouse/client": "^1.11.0", "@esbuild-plugins/node-modules-polyfill": "^0.2.2", "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch", "@langchain/mistralai": "^0.0.24", diff --git a/packages/twenty-server/project.json b/packages/twenty-server/project.json index 58937ac29..09b1b9edb 100644 --- a/packages/twenty-server/project.json +++ b/packages/twenty-server/project.json @@ -204,6 +204,20 @@ }, "defaultConfiguration": "seed" }, + "clickhouse:migrate": { + "executor": "nx:run-commands", + "options": { + "cwd": "packages/twenty-server", + "command": "nx ts-node-no-deps -- src/database/clickhouse/migrations/run-migrations.ts" + } + }, + "clickhouse:seed": { + "executor": "nx:run-commands", + "options": { + "cwd": "packages/twenty-server", + "command": "nx ts-node-no-deps -- src/database/clickhouse/seeds/run-seeds.ts" + } + }, "lingui:extract": { "executor": "nx:run-commands", "options": { diff --git a/packages/twenty-server/src/database/clickhouse/migrations/001-create-events-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/001-create-events-table.sql new file mode 100644 index 000000000..1e6228c70 --- /dev/null +++ b/packages/twenty-server/src/database/clickhouse/migrations/001-create-events-table.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS events +( + `event` LowCardinality(String), + `timestamp` DateTime64(3), + `userId` String DEFAULT '', + `workspaceId` String DEFAULT '', + `properties` JSON +) + ENGINE = MergeTree + ORDER BY (event, workspaceId, timestamp); \ No newline at end of file diff --git a/packages/twenty-server/src/database/clickhouse/migrations/002-create-pageview-table.sql b/packages/twenty-server/src/database/clickhouse/migrations/002-create-pageview-table.sql new file mode 100644 index 000000000..6ed0959af --- /dev/null +++ b/packages/twenty-server/src/database/clickhouse/migrations/002-create-pageview-table.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS pageview +( + `name` LowCardinality(String), + `timestamp` DateTime64(3), + `properties` JSON, + `userId` String DEFAULT '', + `workspaceId` String DEFAULT '' +) + ENGINE = MergeTree + ORDER BY (name, workspaceId, userId, timestamp); diff --git a/packages/twenty-server/src/database/clickhouse/migrations/run-migrations.ts b/packages/twenty-server/src/database/clickhouse/migrations/run-migrations.ts new file mode 100644 index 000000000..f824fb4bb --- /dev/null +++ b/packages/twenty-server/src/database/clickhouse/migrations/run-migrations.ts @@ -0,0 +1,107 @@ +/* eslint-disable no-console */ +import fs from 'fs'; +import path from 'path'; + +import { createClient, ClickHouseClient } from '@clickhouse/client'; +import { config } from 'dotenv'; + +config({ + path: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', + override: true, +}); + +const clickhouseUrl = () => { + const url = process.env.CLICKHOUSE_URL; + + if (url) return url; + + throw new Error( + 'CLICKHOUSE_URL environment variable is not set. Please set it to the ClickHouse URL.', + ); +}; + +async function ensureDatabaseExists() { + const [url, database] = clickhouseUrl().split(/\/(?=[^/]*$)/); + const client = createClient({ + url, + }); + + await client.command({ + query: `CREATE DATABASE IF NOT EXISTS "${database}"`, + }); + await client.command({ + query: `SET enable_json_type = 1`, + }); + + await client.close(); +} + +async function ensureMigrationTable(client: ClickHouseClient) { + await client.command({ + query: ` + CREATE TABLE IF NOT EXISTS migrations ( + filename String, + applied_at DateTime DEFAULT now() + ) ENGINE = MergeTree() + ORDER BY filename; + `, + }); +} + +async function hasMigrationBeenRun( + filename: string, + client: ClickHouseClient, +): Promise { + const resultSet = await client.query({ + query: `SELECT count() as count FROM migrations WHERE filename = {filename:String}`, + query_params: { filename }, + format: 'JSON', + }); + const result = await resultSet.json<{ count: number }>(); + + return result.data[0].count > 0; +} + +async function recordMigration(filename: string, client: ClickHouseClient) { + await client.insert({ + table: 'migrations', + values: [{ filename }], + format: 'JSONEachRow', + }); +} + +async function runMigrations() { + const dir = path.join(__dirname); + const files = fs.readdirSync(dir).filter((f) => f.endsWith('.sql')); + + await ensureDatabaseExists(); + + const client = createClient({ + url: clickhouseUrl(), + }); + + await ensureMigrationTable(client); + + for (const file of files) { + const alreadyRun = await hasMigrationBeenRun(file, client); + + if (alreadyRun) { + console.log(`✔︎ Skipping already applied migration: ${file}`); + continue; + } + + const sql = fs.readFileSync(path.join(dir, file), 'utf8'); + + console.log(`⚡ Running ${file}...`); + await client.command({ query: sql }); + await recordMigration(file, client); + } + + console.log('✅ All migrations applied.'); + await client.close(); +} + +runMigrations().catch((err) => { + console.error('Migration error:', err); + process.exit(1); +}); diff --git a/packages/twenty-server/src/database/clickhouse/seeds/run-seeds.ts b/packages/twenty-server/src/database/clickhouse/seeds/run-seeds.ts new file mode 100644 index 000000000..547ab16e6 --- /dev/null +++ b/packages/twenty-server/src/database/clickhouse/seeds/run-seeds.ts @@ -0,0 +1,38 @@ +/* eslint-disable no-console */ +import { createClient } from '@clickhouse/client'; +import { config } from 'dotenv'; + +import { fixtures } from 'src/engine/core-modules/analytics/utils/fixtures/fixtures'; + +config({ + path: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', + override: true, +}); + +const client = createClient({ + url: process.env.CLICKHOUSE_URL, +}); + +async function seedEvents() { + try { + console.log(`⚡ Seeding ${fixtures.length} events...`); + + await client.insert({ + table: 'events', + values: fixtures, + format: 'JSONEachRow', + }); + + console.log('✅ All events seeded successfully'); + } catch (error) { + console.error('Error seeding events:', error); + throw error; + } finally { + await client.close(); + } +} + +seedEvents().catch((err) => { + console.error('Seeding error:', err); + process.exit(1); +}); diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts index 304d78114..b83967184 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts @@ -1,13 +1,12 @@ import { Injectable } from '@nestjs/common'; -import { OnDatabaseBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-database-batch-event.decorator'; -import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action'; -import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; import { ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event'; import { TelemetryService } from 'src/engine/core-modules/telemetry/telemetry.service'; import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event.type'; -import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants'; import { OnCustomBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-custom-batch-event.decorator'; +import { USER_SIGNUP_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/user/user-signup'; +import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants'; @Injectable() export class TelemetryListener { @@ -16,36 +15,18 @@ export class TelemetryListener { private readonly telemetryService: TelemetryService, ) {} - @OnDatabaseBatchEvent('*', DatabaseEventAction.CREATED) - async handleAllCreate(payload: WorkspaceEventBatch) { - await Promise.all( - payload.events.map((eventPayload) => - this.analyticsService.create( - { - action: payload.name, - payload: {}, - }, - eventPayload.userId, - payload.workspaceId, - ), - ), - ); - } - @OnCustomBatchEvent(USER_SIGNUP_EVENT_NAME) async handleUserSignup( payload: WorkspaceEventBatch, ) { await Promise.all( payload.events.map(async (eventPayload) => { - this.analyticsService.create( - { - action: USER_SIGNUP_EVENT_NAME, - payload: {}, - }, - eventPayload.userId, - payload.workspaceId, - ); + this.analyticsService + .createAnalyticsContext({ + userId: eventPayload.userId, + workspaceId: payload.workspaceId, + }) + .track(USER_SIGNUP_EVENT, {}); this.telemetryService.create( { diff --git a/packages/twenty-server/src/engine/core-modules/analytics/README.md b/packages/twenty-server/src/engine/core-modules/analytics/README.md new file mode 100644 index 000000000..9934e7dc6 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/README.md @@ -0,0 +1,143 @@ +# Analytics Module + +This module provides analytics tracking functionality for the Twenty application. + +## Usage + +### Tracking Events + +The `AnalyticsService` provides a `createAnalyticsContext` method that returns an object with a `track` method. The `track` method is used to track events. + +```typescript +import { Injectable } from '@nestjs/common'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; +import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated'; + +@Injectable() +export class MyService { + constructor(private readonly analyticsService: AnalyticsService) {} + + async doSomething() { + // Create an analytics context + const analytics = this.analyticsService.createAnalyticsContext({ + workspaceId: 'workspace-id', + userId: 'user-id', + }); + + // Track an event + // The event name will be autocompleted + // The properties will be type-checked based on the event name + analytics.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {}); + } +} +``` + +### Adding New Events + +To add a new event: + +1. Create a new file in the `src/engine/core-modules/analytics/utils/events/track` directory +2. Define the event name, schema, and type +3. Register the event using the `registerEvent` function +4. Update the `TrackEventName` and `TrackEventProperties` types in `src/engine/core-modules/analytics/utils/events/event-types.ts` + +Example: + +```typescript +// src/engine/core-modules/analytics/utils/events/track/my-feature/my-event.ts +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const MY_EVENT = 'My Event' as const; +export const myEventSchema = z.object({ + event: z.literal(MY_EVENT), + properties: z.object({ + myProperty: z.string(), + }), +}); + +export type MyEventTrackEvent = z.infer; + +registerEvent(MY_EVENT, myEventSchema); +``` + +Then update the `events.type.ts` file: + +```typescript +// src/engine/core-modules/analytics/types/events.type.ts +import { MY_EVENT, MyEventTrackEvent } from '../utils/events/track/my-feature/my-event'; + +// Add to the union type +export type TrackEventName = + | typeof MY_EVENT + // ... other event names; + +// Add to the TrackEvents interface +export interface TrackEvents { + [MY_EVENT]: MyEventTrackEvent; + // ... other event types +} + +// The TrackEventProperties type will automatically use the new event +export type TrackEventProperties = T extends keyof TrackEvents + ? TrackEvents[T]['properties'] + : object; +``` + +## API + +### AnalyticsService + +#### createAnalyticsContext(context?) + +Creates an analytics context with the given user ID and workspace ID. + +- `context` (optional): An object with `userId` and `workspaceId` properties + +Returns an object with the following methods: + +- `track(event: T, properties: TrackEventProperties)`: Tracks an event with the given name and properties +- `pageview(name: string, properties: Partial)`: Tracks a pageview with the given name and properties + +### Types + +#### TrackEventName + +A union type of all registered event names, plus `string` for backward compatibility. + +#### TrackEventProperties + +A mapped type that maps each event name to its corresponding properties type. It uses the `TrackEvents` interface to provide a more maintainable and type-safe way to map event names to their properties. + +```typescript +// Define the mapping between event names and their event types +export interface TrackEvents { + [EVENT_NAME_1]: Event1Type; + [EVENT_NAME_2]: Event2Type; + // ... other event types +} + +// Use the mapping to extract properties for each event type +export type TrackEventProperties = T extends keyof TrackEvents + ? TrackEvents[T]['properties'] + : object; +``` + +This approach makes it easier to add new events without having to modify a complex nested conditional type. + +#### PageviewProperties + +A type that defines the structure of pageview properties: + +```typescript +type PageviewProperties = { + href: string; + locale: string; + pathname: string; + referrer: string; + sessionId: string; + timeZone: string; + userAgent: string; +}; +``` diff --git a/packages/twenty-server/src/engine/core-modules/analytics/analytics.exception.ts b/packages/twenty-server/src/engine/core-modules/analytics/analytics.exception.ts new file mode 100644 index 000000000..0baeb5ac7 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/analytics.exception.ts @@ -0,0 +1,12 @@ +import { CustomException } from 'src/utils/custom-exception'; + +export class AnalyticsException extends CustomException { + constructor(message: string, code: AnalyticsExceptionCode) { + super(message, code); + } +} + +export enum AnalyticsExceptionCode { + INVALID_TYPE = 'INVALID_TYPE', + INVALID_INPUT = 'INVALID_INPUT', +} diff --git a/packages/twenty-server/src/engine/core-modules/analytics/analytics.module.ts b/packages/twenty-server/src/engine/core-modules/analytics/analytics.module.ts index 95218aeb5..84a751759 100644 --- a/packages/twenty-server/src/engine/core-modules/analytics/analytics.module.ts +++ b/packages/twenty-server/src/engine/core-modules/analytics/analytics.module.ts @@ -1,12 +1,14 @@ import { Module } from '@nestjs/common'; import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module'; +import { ClickhouseService } from 'src/engine/core-modules/analytics/services/clickhouse.service'; import { AnalyticsResolver } from './analytics.resolver'; -import { AnalyticsService } from './analytics.service'; + +import { AnalyticsService } from './services/analytics.service'; @Module({ - providers: [AnalyticsResolver, AnalyticsService], + providers: [AnalyticsResolver, AnalyticsService, ClickhouseService], imports: [JwtModule], exports: [AnalyticsService], }) diff --git a/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.spec.ts b/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.spec.ts index 3d700198e..b9ba87de7 100644 --- a/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.spec.ts @@ -1,18 +1,31 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { User } from 'src/engine/core-modules/user/user.entity'; +import { + AnalyticsException, + AnalyticsExceptionCode, +} from 'src/engine/core-modules/analytics/analytics.exception'; + import { AnalyticsResolver } from './analytics.resolver'; -import { AnalyticsService } from './analytics.service'; + +import { AnalyticsService } from './services/analytics.service'; describe('AnalyticsResolver', () => { let resolver: AnalyticsResolver; + let analyticsService: jest.Mocked; beforeEach(async () => { + analyticsService = { + createAnalyticsContext: jest.fn(), + } as any; + const module: TestingModule = await Test.createTestingModule({ providers: [ AnalyticsResolver, { provide: AnalyticsService, - useValue: {}, + useValue: analyticsService, }, ], }).compile(); @@ -23,4 +36,71 @@ describe('AnalyticsResolver', () => { it('should be defined', () => { expect(resolver).toBeDefined(); }); + + it('should handle a valid pageview input', async () => { + const mockPageview = jest.fn().mockResolvedValue('Pageview created'); + + analyticsService.createAnalyticsContext.mockReturnValue({ + pageview: mockPageview, + track: jest.fn(), + }); + + const input = { + type: 'pageview' as const, + name: 'Test Page', + properties: {}, + }; + const result = await resolver.trackAnalytics( + input, + { id: 'workspace-1' } as Workspace, + { id: 'user-1' } as User, + ); + + expect(analyticsService.createAnalyticsContext).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + userId: 'user-1', + }); + expect(mockPageview).toHaveBeenCalledWith('Test Page', {}); + expect(result).toBe('Pageview created'); + }); + + it('should handle a valid track input', async () => { + const mockTrack = jest.fn().mockResolvedValue('Track created'); + + analyticsService.createAnalyticsContext.mockReturnValue({ + track: mockTrack, + pageview: jest.fn(), + }); + + const input = { + type: 'track' as const, + event: 'Custom Domain Activated' as const, + properties: {}, + }; + const result = await resolver.trackAnalytics( + input, + { id: 'workspace-2' } as Workspace, + { id: 'user-2' } as User, + ); + + expect(analyticsService.createAnalyticsContext).toHaveBeenCalledWith({ + workspaceId: 'workspace-2', + userId: 'user-2', + }); + expect(mockTrack).toHaveBeenCalledWith('Custom Domain Activated', {}); + expect(result).toBe('Track created'); + }); + + it('should throw an AnalyticsException for invalid input', async () => { + const invalidInput = { type: 'invalid' }; + + await expect( + resolver.trackAnalytics(invalidInput as any, undefined, undefined), + ).rejects.toThrowError( + new AnalyticsException( + 'Invalid analytics input', + AnalyticsExceptionCode.INVALID_TYPE, + ), + ); + }); }); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.ts b/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.ts index 4fe36e677..4e2fa3ffc 100644 --- a/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/analytics/analytics.resolver.ts @@ -4,26 +4,63 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator'; import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; +import { + AnalyticsException, + AnalyticsExceptionCode, +} from 'src/engine/core-modules/analytics/analytics.exception'; -import { AnalyticsService } from './analytics.service'; - -import { CreateAnalyticsInput } from './dtos/create-analytics.input'; +import { AnalyticsService } from './services/analytics.service'; +import { + CreateAnalyticsInput, + CreateAnalyticsInputV2, + isPageviewAnalyticsInput, + isTrackAnalyticsInput, +} from './dtos/create-analytics.input'; import { Analytics } from './entities/analytics.entity'; @Resolver(() => Analytics) export class AnalyticsResolver { constructor(private readonly analyticsService: AnalyticsService) {} + // deprecated @Mutation(() => Analytics) track( - @Args() createAnalyticsInput: CreateAnalyticsInput, + @Args() _createAnalyticsInput: CreateAnalyticsInput, + @AuthWorkspace() _workspace: Workspace | undefined, + @AuthUser({ allowUndefined: true }) _user: User | undefined, + ) { + return { success: true }; + } + + @Mutation(() => Analytics) + async trackAnalytics( + @Args() + createAnalyticsInput: CreateAnalyticsInputV2, @AuthWorkspace() workspace: Workspace | undefined, @AuthUser({ allowUndefined: true }) user: User | undefined, ) { - return this.analyticsService.create( - createAnalyticsInput, - user?.id, - workspace?.id, + const analyticsContext = this.analyticsService.createAnalyticsContext({ + workspaceId: workspace?.id, + userId: user?.id, + }); + + if (isPageviewAnalyticsInput(createAnalyticsInput)) { + return analyticsContext.pageview( + createAnalyticsInput.name, + createAnalyticsInput.properties ?? {}, + ); + } + + if (isTrackAnalyticsInput(createAnalyticsInput)) { + return analyticsContext.track( + createAnalyticsInput.event, + createAnalyticsInput.properties ?? {}, + ); + } + + throw new AnalyticsException( + 'Invalid analytics input', + AnalyticsExceptionCode.INVALID_TYPE, ); } } diff --git a/packages/twenty-server/src/engine/core-modules/analytics/analytics.service.spec.ts b/packages/twenty-server/src/engine/core-modules/analytics/analytics.service.spec.ts deleted file mode 100644 index c4818ff2c..000000000 --- a/packages/twenty-server/src/engine/core-modules/analytics/analytics.service.spec.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { HttpService } from '@nestjs/axios'; -import { Test, TestingModule } from '@nestjs/testing'; - -import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service'; -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; - -import { AnalyticsService } from './analytics.service'; - -describe('AnalyticsService', () => { - let service: AnalyticsService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - AnalyticsService, - { - provide: TwentyConfigService, - useValue: {}, - }, - { - provide: JwtWrapperService, - useValue: {}, - }, - { - provide: HttpService, - useValue: {}, - }, - ], - }).compile(); - - service = module.get(AnalyticsService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/analytics.service.ts b/packages/twenty-server/src/engine/core-modules/analytics/analytics.service.ts index 01e3fcebc..e69de29bb 100644 --- a/packages/twenty-server/src/engine/core-modules/analytics/analytics.service.ts +++ b/packages/twenty-server/src/engine/core-modules/analytics/analytics.service.ts @@ -1,53 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; - -type CreateEventInput = { - action: string; - payload: object; -}; - -@Injectable() -export class AnalyticsService { - constructor(private readonly twentyConfigService: TwentyConfigService) {} - - async create( - createEventInput: CreateEventInput, - userId: string | null | undefined, - workspaceId: string | null | undefined, - ) { - if (!this.twentyConfigService.get('ANALYTICS_ENABLED')) { - return { success: true }; - } - - let _data; - - switch (createEventInput.action) { - case 'pageview': - _data = { - timestamp: new Date().toISOString(), - version: '1', - userId: userId, - workspaceId: workspaceId, - ...createEventInput.payload, - }; - break; - default: - _data = { - action: createEventInput.action, - timestamp: new Date().toISOString(), - version: '1', - userId: userId, - workspaceId: workspaceId, - payload: { - ...createEventInput.payload, - }, - }; - break; - } - - // TODO: send event to clickhouse - - return { success: true }; - } -} diff --git a/packages/twenty-server/src/engine/core-modules/analytics/dtos/create-analytics.input.ts b/packages/twenty-server/src/engine/core-modules/analytics/dtos/create-analytics.input.ts index 5e887dca0..c25874e9f 100644 --- a/packages/twenty-server/src/engine/core-modules/analytics/dtos/create-analytics.input.ts +++ b/packages/twenty-server/src/engine/core-modules/analytics/dtos/create-analytics.input.ts @@ -1,8 +1,27 @@ -import { ArgsType, Field } from '@nestjs/graphql'; +import { ArgsType, Field, registerEnumType } from '@nestjs/graphql'; -import { IsNotEmpty, IsObject, IsString } from 'class-validator'; -import graphqlTypeJson from 'graphql-type-json'; +import { + IsEnum, + IsNotEmpty, + IsObject, + IsOptional, + IsString, +} from 'class-validator'; +import GraphQLJSON from 'graphql-type-json'; +import { TrackEventName } from 'src/engine/core-modules/analytics/types/events.type'; +import { PageviewProperties } from 'src/engine/core-modules/analytics/utils/events/pageview/pageview'; + +enum AnalyticsType { + PAGEVIEW = 'pageview', + TRACK = 'track', +} + +registerEnumType(AnalyticsType, { + name: 'AnalyticsType', +}); + +// deprecated @ArgsType() export class CreateAnalyticsInput { @Field({ description: 'Type of the event' }) @@ -10,7 +29,41 @@ export class CreateAnalyticsInput { @IsString() action: string; - @Field(() => graphqlTypeJson, { description: 'Event payload in JSON format' }) + @Field(() => GraphQLJSON, { description: 'Event payload in JSON format' }) @IsObject() payload: JSON; } + +@ArgsType() +export class CreateAnalyticsInputV2 { + @Field(() => AnalyticsType) + @IsEnum(AnalyticsType) + type: 'pageview' | 'track'; + + @Field(() => String, { nullable: true }) + @IsOptional() + @IsString() + name?: string; + + @Field(() => String, { nullable: true }) + @IsOptional() + @IsString() + event?: TrackEventName; + + @Field(() => GraphQLJSON, { nullable: true }) + @IsOptional() + @IsObject() + properties?: PageviewProperties | Record; +} + +export function isPageviewAnalyticsInput( + input: CreateAnalyticsInputV2, +): input is CreateAnalyticsInputV2 & { name: string } { + return input.type === 'pageview' && !!input.name; +} + +export function isTrackAnalyticsInput( + input: CreateAnalyticsInputV2, +): input is CreateAnalyticsInputV2 & { event: TrackEventName } { + return input.type === 'track' && !!input.event; +} diff --git a/packages/twenty-server/src/engine/core-modules/analytics/services/analytics.service.spec.ts b/packages/twenty-server/src/engine/core-modules/analytics/services/analytics.service.spec.ts new file mode 100644 index 000000000..808c92afe --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/services/analytics.service.spec.ts @@ -0,0 +1,127 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { AnalyticsContextMock } from 'test/utils/analytics-context.mock'; + +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { ClickhouseService } from 'src/engine/core-modules/analytics/services/clickhouse.service'; +import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; +import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated'; + +import { AnalyticsService } from './analytics.service'; + +describe('AnalyticsService', () => { + let service: AnalyticsService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + { + provide: AnalyticsService, + useValue: { + createAnalyticsContext: AnalyticsContextMock, + }, + }, + { + provide: TwentyConfigService, + useValue: { + get: jest.fn().mockReturnValue(true), + }, + }, + { + provide: ClickhouseService, + useValue: { + pushEvent: jest.fn(), + }, + }, + { + provide: ExceptionHandlerService, + useValue: { + captureExceptions: jest.fn(), + }, + }, + ], + }).compile(); + + service = module.get(AnalyticsService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('createAnalyticsContext', () => { + const mockUserIdAndWorkspaceId = { + userId: 'test-user-id', + workspaceId: 'test-workspace-id', + }; + + it('should create a valid context object', () => { + const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId); + + expect(context).toHaveProperty('track'); + expect(context).toHaveProperty('pageview'); + }); + + it('should call track with correct parameters', async () => { + const trackSpy = jest.fn().mockResolvedValue({ success: true }); + const mockContext = AnalyticsContextMock({ + track: trackSpy, + }); + + jest + .spyOn(service, 'createAnalyticsContext') + .mockReturnValue(mockContext); + + const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId); + + await context.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {}); + + expect(trackSpy).toHaveBeenCalledWith(CUSTOM_DOMAIN_ACTIVATED_EVENT, {}); + }); + + it('should call pageview with correct parameters', async () => { + const pageviewSpy = jest.fn().mockResolvedValue({ success: true }); + const mockContext = AnalyticsContextMock({ + pageview: pageviewSpy, + }); + + jest + .spyOn(service, 'createAnalyticsContext') + .mockReturnValue(mockContext); + + const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId); + const testPageviewProperties = { + href: '/test-url', + locale: '', + pathname: '', + referrer: '', + sessionId: '', + timeZone: '', + userAgent: '', + }; + + await context.pageview('page-view', testPageviewProperties); + + expect(pageviewSpy).toHaveBeenCalledWith( + 'page-view', + testPageviewProperties, + ); + }); + + it('should return success when track is called', async () => { + const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId); + + const result = await context.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {}); + + expect(result).toEqual({ success: true }); + }); + + it('should return success when pageview is called', async () => { + const context = service.createAnalyticsContext(mockUserIdAndWorkspaceId); + + const result = await context.pageview('page-view', {}); + + expect(result).toEqual({ success: true }); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/services/analytics.service.ts b/packages/twenty-server/src/engine/core-modules/analytics/services/analytics.service.ts new file mode 100644 index 000000000..91ec83c16 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/services/analytics.service.ts @@ -0,0 +1,70 @@ +import { Injectable } from '@nestjs/common'; + +import { + makePageview, + makeTrackEvent, +} from 'src/engine/core-modules/analytics/utils/analytics.utils'; +import { ClickhouseService } from 'src/engine/core-modules/analytics/services/clickhouse.service'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { + TrackEventName, + TrackEventProperties, +} from 'src/engine/core-modules/analytics/types/events.type'; +import { PageviewProperties } from 'src/engine/core-modules/analytics/utils/events/pageview/pageview'; +import { + AnalyticsException, + AnalyticsExceptionCode, +} from 'src/engine/core-modules/analytics/analytics.exception'; + +@Injectable() +export class AnalyticsService { + constructor( + private readonly twentyConfigService: TwentyConfigService, + private readonly clickhouseService: ClickhouseService, + ) {} + + createAnalyticsContext(context?: { + workspaceId?: string | null | undefined; + userId?: string | null | undefined; + }) { + const userIdAndWorkspaceId = context + ? { + ...(context.userId ? { userId: context.userId } : {}), + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + } + : {}; + + return { + track: ( + event: T, + properties: TrackEventProperties, + ) => + this.preventAnalyticsIfDisabled(() => + this.clickhouseService.pushEvent({ + ...userIdAndWorkspaceId, + ...makeTrackEvent(event, properties), + }), + ), + pageview: (name: string, properties: Partial) => + this.preventAnalyticsIfDisabled(() => + this.clickhouseService.pushEvent({ + ...userIdAndWorkspaceId, + ...makePageview(name, properties), + }), + ), + }; + } + + private preventAnalyticsIfDisabled( + sendEventOrPageviewFunction: () => Promise<{ success: boolean }>, + ) { + if (!this.twentyConfigService.get('ANALYTICS_ENABLED')) { + return { success: true }; + } + try { + return sendEventOrPageviewFunction(); + } catch (err) { + return new AnalyticsException(err, AnalyticsExceptionCode.INVALID_INPUT); + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/analytics/services/clickhouse.service.spec.ts b/packages/twenty-server/src/engine/core-modules/analytics/services/clickhouse.service.spec.ts new file mode 100644 index 000000000..b98e14d16 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/services/clickhouse.service.spec.ts @@ -0,0 +1,159 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; +import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated'; +import { + makePageview, + makeTrackEvent, +} from 'src/engine/core-modules/analytics/utils/analytics.utils'; + +import { ClickhouseService } from './clickhouse.service'; + +// Mock the createClient function from @clickhouse/client +jest.mock('@clickhouse/client', () => ({ + createClient: jest.fn().mockReturnValue({ + insert: jest.fn().mockResolvedValue({}), + }), +})); + +describe('ClickhouseService', () => { + let service: ClickhouseService; + let twentyConfigService: TwentyConfigService; + let exceptionHandlerService: ExceptionHandlerService; + let mockClickhouseClient: { insert: jest.Mock }; + + const mockPageview = makePageview('Home', { + href: 'https://example.com/test', + locale: 'en-US', + pathname: '/test', + referrer: 'https://example.com', + sessionId: 'test-session-id', + timeZone: 'UTC', + userAgent: 'test-user-agent', + }); + + const mockEvent = makeTrackEvent(CUSTOM_DOMAIN_ACTIVATED_EVENT, {}); + + beforeEach(async () => { + jest.clearAllMocks(); + + mockClickhouseClient = { + insert: jest.fn().mockResolvedValue({}), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ClickhouseService, + { + provide: TwentyConfigService, + useValue: { + get: jest.fn((key) => { + if (key === 'ANALYTICS_ENABLED') return true; + if (key === 'CLICKHOUSE_URL') return 'http://localhost:8123'; + + return null; + }), + }, + }, + { + provide: ExceptionHandlerService, + useValue: { + captureExceptions: jest.fn(), + }, + }, + ], + }).compile(); + + service = module.get(ClickhouseService); + twentyConfigService = module.get(TwentyConfigService); + exceptionHandlerService = module.get( + ExceptionHandlerService, + ); + + // Set the mock client + // @ts-expect-error accessing private property for testing + service.clickhouseClient = mockClickhouseClient; + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('constructor', () => { + it('should not initialize clickhouse client when analytics is disabled', async () => { + jest.spyOn(twentyConfigService, 'get').mockImplementation((key) => { + if (key === 'ANALYTICS_ENABLED') return false; + }); + + const newModule: TestingModule = await Test.createTestingModule({ + providers: [ + ClickhouseService, + { + provide: TwentyConfigService, + useValue: twentyConfigService, + }, + { + provide: ExceptionHandlerService, + useValue: exceptionHandlerService, + }, + ], + }).compile(); + + const newService = newModule.get(ClickhouseService); + + // @ts-expect-error accessing private property for testing + expect(newService.clickhouseClient).toBeUndefined(); + }); + }); + + describe('pushEvent', () => { + it('should insert event into clickhouse and return success', async () => { + const result = await service.pushEvent(mockEvent); + + expect(result).toEqual({ success: true }); + const { type: _type, ...rest } = mockEvent; + + expect(mockClickhouseClient.insert).toHaveBeenCalledWith({ + table: 'events', + values: [rest], + format: 'JSONEachRow', + }); + }); + + it('should insert pageview into clickhouse and return success', async () => { + const result = await service.pushEvent(mockPageview); + + expect(result).toEqual({ success: true }); + const { type: _type, ...rest } = mockPageview; + + expect(mockClickhouseClient.insert).toHaveBeenCalledWith({ + table: 'pageview', + values: [rest], + format: 'JSONEachRow', + }); + }); + + it('should return success when clickhouse client is not defined', async () => { + // @ts-expect-error accessing private property for testing + service.clickhouseClient = undefined; + + const result = await service.pushEvent(mockEvent); + + expect(result).toEqual({ success: true }); + }); + + it('should handle errors and return failure', async () => { + const testError = new Error('Test error'); + + mockClickhouseClient.insert.mockRejectedValueOnce(testError); + + const result = await service.pushEvent(mockEvent); + + expect(result).toEqual({ success: false }); + expect(exceptionHandlerService.captureExceptions).toHaveBeenCalledWith([ + testError, + ]); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/services/clickhouse.service.ts b/packages/twenty-server/src/engine/core-modules/analytics/services/clickhouse.service.ts new file mode 100644 index 000000000..d4a70ca18 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/services/clickhouse.service.ts @@ -0,0 +1,60 @@ +import { Injectable } from '@nestjs/common'; + +import { ClickHouseClient, createClient } from '@clickhouse/client'; + +import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { + makePageview, + makeTrackEvent, +} from 'src/engine/core-modules/analytics/utils/analytics.utils'; + +@Injectable() +export class ClickhouseService { + private clickhouseClient: ClickHouseClient | undefined; + constructor( + private readonly exceptionHandlerService: ExceptionHandlerService, + private readonly twentyConfigService: TwentyConfigService, + ) { + if (twentyConfigService.get('ANALYTICS_ENABLED')) { + this.clickhouseClient = createClient({ + url: twentyConfigService.get('CLICKHOUSE_URL'), + compression: { + response: true, + request: true, + }, + clickhouse_settings: { + async_insert: 1, + wait_for_async_insert: 1, + }, + }); + } + } + + async pushEvent( + data: ( + | ReturnType + | ReturnType + ) & { userId?: string | null; workspaceId?: string | null }, + ) { + try { + if (!this.clickhouseClient) { + return { success: true }; + } + + const { type, ...rest } = data; + + await this.clickhouseClient.insert({ + table: type === 'page' ? 'pageview' : 'events', + values: [rest], + format: 'JSONEachRow', + }); + + return { success: true }; + } catch (err) { + this.exceptionHandlerService.captureExceptions([err]); + + return { success: false }; + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/analytics/types/common.type.ts b/packages/twenty-server/src/engine/core-modules/analytics/types/common.type.ts new file mode 100644 index 000000000..cc4a2995a --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/types/common.type.ts @@ -0,0 +1,2 @@ +export type AnalyticsCommonPropertiesType = 'timestamp' | 'version'; +export type IdentifierType = 'workspaceId' | 'userId'; diff --git a/packages/twenty-server/src/engine/core-modules/analytics/types/events.type.ts b/packages/twenty-server/src/engine/core-modules/analytics/types/events.type.ts new file mode 100644 index 000000000..c6810587a --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/types/events.type.ts @@ -0,0 +1,72 @@ +import { + WEBHOOK_RESPONSE_EVENT, + WebhookResponseTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/webhook/webhook-response'; +import { + SERVERLESS_FUNCTION_EXECUTED_EVENT, + ServerlessFunctionExecutedTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/serverless-function/serverless-function-executed'; +import { + CUSTOM_DOMAIN_DEACTIVATED_EVENT, + CustomDomainDeactivatedTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated'; +import { + CUSTOM_DOMAIN_ACTIVATED_EVENT, + CustomDomainActivatedTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated'; +import { + WORKSPACE_ENTITY_CREATED_EVENT, + WorkspaceEntityCreatedTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/workspace-entity/workspace-entity-created'; +import { + USER_SIGNUP_EVENT, + UserSignupTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/user/user-signup'; +import { + MONITORING_EVENT, + MonitoringTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/monitoring/monitoring'; +import { + OBJECT_RECORD_CREATED_EVENT, + ObjectRecordCreatedTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created'; +import { + OBJECT_RECORD_UPDATED_EVENT, + ObjectRecordUpdatedTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated'; +import { + OBJECT_RECORD_DELETED_EVENT, + ObjectRecordDeletedTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete'; + +// Define all track event names +export type TrackEventName = + | typeof CUSTOM_DOMAIN_ACTIVATED_EVENT + | typeof CUSTOM_DOMAIN_DEACTIVATED_EVENT + | typeof SERVERLESS_FUNCTION_EXECUTED_EVENT + | typeof WEBHOOK_RESPONSE_EVENT + | typeof WORKSPACE_ENTITY_CREATED_EVENT + | typeof MONITORING_EVENT + | typeof OBJECT_RECORD_CREATED_EVENT + | typeof OBJECT_RECORD_UPDATED_EVENT + | typeof OBJECT_RECORD_DELETED_EVENT + | typeof USER_SIGNUP_EVENT; + +// Map event names to their corresponding event types +export interface TrackEvents { + [CUSTOM_DOMAIN_ACTIVATED_EVENT]: CustomDomainActivatedTrackEvent; + [CUSTOM_DOMAIN_DEACTIVATED_EVENT]: CustomDomainDeactivatedTrackEvent; + [SERVERLESS_FUNCTION_EXECUTED_EVENT]: ServerlessFunctionExecutedTrackEvent; + [WEBHOOK_RESPONSE_EVENT]: WebhookResponseTrackEvent; + [WORKSPACE_ENTITY_CREATED_EVENT]: WorkspaceEntityCreatedTrackEvent; + [USER_SIGNUP_EVENT]: UserSignupTrackEvent; + [MONITORING_EVENT]: MonitoringTrackEvent; + [OBJECT_RECORD_DELETED_EVENT]: ObjectRecordDeletedTrackEvent; + [OBJECT_RECORD_CREATED_EVENT]: ObjectRecordCreatedTrackEvent; + [OBJECT_RECORD_UPDATED_EVENT]: ObjectRecordUpdatedTrackEvent; +} + +export type TrackEventProperties = + T extends keyof TrackEvents + ? TrackEvents[T]['properties'] + : Record; diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/analytics.utils.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/analytics.utils.ts new file mode 100644 index 000000000..eb63c5a8f --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/analytics.utils.ts @@ -0,0 +1,50 @@ +import { format } from 'date-fns'; + +import { AnalyticsCommonPropertiesType } from 'src/engine/core-modules/analytics/types/common.type'; +import { + PageviewProperties, + pageviewSchema, +} from 'src/engine/core-modules/analytics/utils/events/pageview/pageview'; +import { + TrackEventName, + TrackEventProperties, +} from 'src/engine/core-modules/analytics/types/events.type'; +import { + eventsRegistry, + GenericTrackEvent, +} from 'src/engine/core-modules/analytics/utils/events/track/track'; + +const common = (): Record => ({ + timestamp: format(new Date(), 'yyyy-MM-dd HH:mm:ss'), + version: '1', +}); + +export function makePageview( + name: string, + properties: Partial = {}, +) { + return pageviewSchema.parse({ + type: 'page', + name, + ...common(), + properties, + }); +} + +export function makeTrackEvent( + event: T, + properties: TrackEventProperties, +): GenericTrackEvent { + const schema = eventsRegistry.get(event); + + if (!schema) { + throw new Error(`Schema for event ${event} is not implemented`); + } + + return schema.parse({ + type: 'track', + event, + properties, + ...common(), + }); +} diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/common/base-schemas.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/common/base-schemas.ts new file mode 100644 index 000000000..e56283daf --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/common/base-schemas.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; + +export const baseEventSchema = z + .object({ + timestamp: z.string(), + userId: z.string().nullish(), + workspaceId: z.string().nullish(), + version: z.string(), + }) + .strict(); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/pageview/pageview.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/pageview/pageview.ts new file mode 100644 index 000000000..a8c740aa0 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/pageview/pageview.ts @@ -0,0 +1,19 @@ +import { z } from 'zod'; + +import { baseEventSchema } from 'src/engine/core-modules/analytics/utils/events/common/base-schemas'; + +export const pageviewSchema = baseEventSchema.extend({ + type: z.literal('page'), + name: z.string(), + properties: z.object({ + href: z.string(), + locale: z.string(), + pathname: z.string(), + referrer: z.string(), + sessionId: z.string(), + timeZone: z.string(), + userAgent: z.string(), + }), +}); + +export type PageviewProperties = z.infer['properties']; diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated.ts new file mode 100644 index 000000000..bb2480b19 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated.ts @@ -0,0 +1,17 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const CUSTOM_DOMAIN_ACTIVATED_EVENT = 'Custom Domain Activated' as const; +export const customDomainActivatedSchema = z + .object({ + event: z.literal(CUSTOM_DOMAIN_ACTIVATED_EVENT), + properties: z.object({}).strict(), + }) + .strict(); + +export type CustomDomainActivatedTrackEvent = z.infer< + typeof customDomainActivatedSchema +>; + +registerEvent(CUSTOM_DOMAIN_ACTIVATED_EVENT, customDomainActivatedSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated.ts new file mode 100644 index 000000000..c4d04d820 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated.ts @@ -0,0 +1,18 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const CUSTOM_DOMAIN_DEACTIVATED_EVENT = + 'Custom Domain Deactivated' as const; +export const customDomainDeactivatedSchema = z + .object({ + event: z.literal(CUSTOM_DOMAIN_DEACTIVATED_EVENT), + properties: z.object({}).strict(), + }) + .strict(); + +export type CustomDomainDeactivatedTrackEvent = z.infer< + typeof customDomainDeactivatedSchema +>; + +registerEvent(CUSTOM_DOMAIN_DEACTIVATED_EVENT, customDomainDeactivatedSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/monitoring/monitoring.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/monitoring/monitoring.ts new file mode 100644 index 000000000..59f340058 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/monitoring/monitoring.ts @@ -0,0 +1,22 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const MONITORING_EVENT = 'Monitoring' as const; +export const monitoringSchema = z + .object({ + event: z.literal(MONITORING_EVENT), + properties: z + .object({ + eventName: z.string(), + connectedAccountId: z.string().optional(), + messageChannelId: z.string().optional(), + message: z.string().optional(), + }) + .strict(), + }) + .strict(); + +export type MonitoringTrackEvent = z.infer; + +registerEvent(MONITORING_EVENT, monitoringSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created.ts new file mode 100644 index 000000000..55bfe4f30 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created.ts @@ -0,0 +1,15 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const OBJECT_RECORD_CREATED_EVENT = 'Object Record Created' as const; +export const objectRecordCreatedSchema = z.object({ + event: z.literal(OBJECT_RECORD_CREATED_EVENT), + properties: z.object({}).passthrough(), +}); + +export type ObjectRecordCreatedTrackEvent = z.infer< + typeof objectRecordCreatedSchema +>; + +registerEvent(OBJECT_RECORD_CREATED_EVENT, objectRecordCreatedSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete.ts new file mode 100644 index 000000000..2865d8e1b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete.ts @@ -0,0 +1,15 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const OBJECT_RECORD_DELETED_EVENT = 'Object Record Deleted' as const; +export const objectRecordDeletedSchema = z.object({ + event: z.literal(OBJECT_RECORD_DELETED_EVENT), + properties: z.object({}).passthrough(), +}); + +export type ObjectRecordDeletedTrackEvent = z.infer< + typeof objectRecordDeletedSchema +>; + +registerEvent(OBJECT_RECORD_DELETED_EVENT, objectRecordDeletedSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated.ts new file mode 100644 index 000000000..4d34ffddb --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated.ts @@ -0,0 +1,15 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const OBJECT_RECORD_UPDATED_EVENT = 'Object Record Updated' as const; +export const objectRecordUpdatedSchema = z.object({ + event: z.literal(OBJECT_RECORD_UPDATED_EVENT), + properties: z.object({}).passthrough(), +}); + +export type ObjectRecordUpdatedTrackEvent = z.infer< + typeof objectRecordUpdatedSchema +>; + +registerEvent(OBJECT_RECORD_UPDATED_EVENT, objectRecordUpdatedSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/serverless-function/serverless-function-executed.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/serverless-function/serverless-function-executed.ts new file mode 100644 index 000000000..dd0a560bc --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/serverless-function/serverless-function-executed.ts @@ -0,0 +1,29 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const SERVERLESS_FUNCTION_EXECUTED_EVENT = + 'Serverless Function Executed' as const; +export const serverlessFunctionExecutedSchema = z + .object({ + event: z.literal(SERVERLESS_FUNCTION_EXECUTED_EVENT), + properties: z + .object({ + duration: z.number(), + status: z.enum(['IDLE', 'SUCCESS', 'ERROR']), + errorType: z.string().optional(), + functionId: z.string(), + functionName: z.string(), + }) + .strict(), + }) + .strict(); + +export type ServerlessFunctionExecutedTrackEvent = z.infer< + typeof serverlessFunctionExecutedSchema +>; + +registerEvent( + SERVERLESS_FUNCTION_EXECUTED_EVENT, + serverlessFunctionExecutedSchema, +); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/track.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/track.ts new file mode 100644 index 000000000..54eb7cdf7 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/track.ts @@ -0,0 +1,28 @@ +import { z } from 'zod'; + +import { baseEventSchema } from 'src/engine/core-modules/analytics/utils/events/common/base-schemas'; + +export const genericTrackSchema = baseEventSchema.extend({ + type: z.literal('track'), + event: z.string(), + properties: z.any(), +}); + +export type GenericTrackEvent = { + type: 'track'; + event: E; + properties: any; + timestamp: string; + version: string; + userId?: string; + workspaceId?: string; +}; + +export const eventsRegistry = new Map>(); + +export function registerEvent>( + event: E, + schema: S, +): void { + eventsRegistry.set(event, genericTrackSchema.merge(schema)); +} diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/user/user-signup.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/user/user-signup.ts new file mode 100644 index 000000000..53c97caaf --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/user/user-signup.ts @@ -0,0 +1,15 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const USER_SIGNUP_EVENT = 'User Signup' as const; +export const userSignupSchema = z + .object({ + event: z.literal(USER_SIGNUP_EVENT), + properties: z.object({}).strict(), + }) + .strict(); + +export type UserSignupTrackEvent = z.infer; + +registerEvent(USER_SIGNUP_EVENT, userSignupSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/webhook/webhook-response.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/webhook/webhook-response.ts new file mode 100644 index 000000000..81b6d070d --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/webhook/webhook-response.ts @@ -0,0 +1,23 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const WEBHOOK_RESPONSE_EVENT = 'Webhook Response' as const; +export const webhookResponseSchema = z + .object({ + event: z.literal(WEBHOOK_RESPONSE_EVENT), + properties: z + .object({ + status: z.number().optional(), + success: z.boolean(), + url: z.string(), + webhookId: z.string(), + eventName: z.string(), + }) + .strict(), + }) + .strict(); + +export type WebhookResponseTrackEvent = z.infer; + +registerEvent(WEBHOOK_RESPONSE_EVENT, webhookResponseSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/workspace-entity/workspace-entity-created.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/workspace-entity/workspace-entity-created.ts new file mode 100644 index 000000000..28b2ac87e --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/events/track/workspace-entity/workspace-entity-created.ts @@ -0,0 +1,20 @@ +import { z } from 'zod'; + +import { registerEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; + +export const WORKSPACE_ENTITY_CREATED_EVENT = + 'Workspace Entity Created' as const; +export const workspaceEntityCreatedSchema = z + .object({ + event: z.literal(WORKSPACE_ENTITY_CREATED_EVENT), + properties: z.object({ + name: z.string(), + }), + }) + .strict(); + +export type WorkspaceEntityCreatedTrackEvent = z.infer< + typeof workspaceEntityCreatedSchema +>; + +registerEvent(WORKSPACE_ENTITY_CREATED_EVENT, workspaceEntityCreatedSchema); diff --git a/packages/twenty-server/src/engine/core-modules/analytics/utils/fixtures/fixtures.ts b/packages/twenty-server/src/engine/core-modules/analytics/utils/fixtures/fixtures.ts new file mode 100644 index 000000000..d9620939d --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/analytics/utils/fixtures/fixtures.ts @@ -0,0 +1,54 @@ +import { GenericTrackEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; +import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created'; +import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated'; +import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated'; +import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated'; +import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete'; + +export const fixtures: Array = [ + { + type: 'track', + event: CUSTOM_DOMAIN_ACTIVATED_EVENT, + timestamp: '2024-10-24T15:55:35.177', + version: '1', + userId: '20202020-9e3b-46d4-a556-88b9ddc2b034', + workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419', + properties: {}, + }, + { + type: 'track', + event: CUSTOM_DOMAIN_DEACTIVATED_EVENT, + timestamp: '2024-10-24T15:55:35.177', + version: '1', + userId: '20202020-9e3b-46d4-a556-88b9ddc2b034', + workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419', + properties: {}, + }, + { + type: 'track', + event: OBJECT_RECORD_CREATED_EVENT, + timestamp: '2024-10-24T15:55:35.177', + version: '1', + userId: '20202020-9e3b-46d4-a556-88b9ddc2b034', + workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419', + properties: {}, + }, + { + type: 'track', + event: OBJECT_RECORD_UPDATED_EVENT, + timestamp: '2024-10-24T15:55:35.177', + version: '1', + userId: '20202020-9e3b-46d4-a556-88b9ddc2b034', + workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419', + properties: {}, + }, + { + type: 'track', + event: OBJECT_RECORD_DELETED_EVENT, + timestamp: '2024-10-24T15:55:35.177', + version: '1', + userId: '20202020-9e3b-46d4-a556-88b9ddc2b034', + workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419', + properties: {}, + }, +]; diff --git a/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.controller.ts b/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.controller.ts index ae8dd44c6..ba30a1aec 100644 --- a/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.controller.ts +++ b/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.controller.ts @@ -24,6 +24,8 @@ import { handleException } from 'src/engine/core-modules/exception-handler/http- import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service'; import { CloudflareSecretMatchGuard } from 'src/engine/core-modules/domain-manager/guards/cloudflare-secret.guard'; import { CustomDomainService } from 'src/engine/core-modules/domain-manager/services/custom-domain.service'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; +import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated'; @Controller('cloudflare') @UseFilters(AuthRestApiExceptionFilter) @@ -34,6 +36,7 @@ export class CloudflareController { private readonly domainManagerService: DomainManagerService, private readonly customDomainService: CustomDomainService, private readonly exceptionHandlerService: ExceptionHandlerService, + private readonly analyticsService: AnalyticsService, ) {} @Post('custom-hostname-webhooks') @@ -57,6 +60,10 @@ export class CloudflareController { if (!workspace) return; + const analytics = this.analyticsService.createAnalyticsContext({ + workspaceId: workspace.id, + }); + const customDomainDetails = await this.customDomainService.getCustomDomainDetails( req.body.data.data.hostname, @@ -83,6 +90,8 @@ export class CloudflareController { ...workspace, ...workspaceUpdated, }); + + await analytics.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {}); } return res.status(200).send(); diff --git a/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.spec.ts b/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.spec.ts index 9ed483f30..7a12e88f2 100644 --- a/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.spec.ts @@ -3,6 +3,7 @@ import { getRepositoryToken } from '@nestjs/typeorm'; import { Request, Response } from 'express'; import { Repository } from 'typeorm'; +import { AnalyticsContextMock } from 'test/utils/analytics-context.mock'; import { CloudflareController } from 'src/engine/core-modules/domain-manager/controllers/cloudflare.controller'; import { CustomDomainValidRecords } from 'src/engine/core-modules/domain-manager/dtos/custom-domain-valid-records'; @@ -12,6 +13,7 @@ import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handl import { HttpExceptionHandlerService } from 'src/engine/core-modules/exception-handler/http-exception-handler.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; describe('CloudflareController - customHostnameWebhooks', () => { let controller: CloudflareController; @@ -61,6 +63,12 @@ describe('CloudflareController - customHostnameWebhooks', () => { get: jest.fn(), }, }, + { + provide: AnalyticsService, + useValue: { + createAnalyticsContext: AnalyticsContextMock, + }, + }, ], }).compile(); diff --git a/packages/twenty-server/src/engine/core-modules/domain-manager/domain-manager.module.ts b/packages/twenty-server/src/engine/core-modules/domain-manager/domain-manager.module.ts index 5def9ce30..61defa358 100644 --- a/packages/twenty-server/src/engine/core-modules/domain-manager/domain-manager.module.ts +++ b/packages/twenty-server/src/engine/core-modules/domain-manager/domain-manager.module.ts @@ -5,9 +5,10 @@ import { DomainManagerService } from 'src/engine/core-modules/domain-manager/ser import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { CloudflareController } from 'src/engine/core-modules/domain-manager/controllers/cloudflare.controller'; import { CustomDomainService } from 'src/engine/core-modules/domain-manager/services/custom-domain.service'; +import { AnalyticsModule } from 'src/engine/core-modules/analytics/analytics.module'; @Module({ - imports: [TypeOrmModule.forFeature([Workspace], 'core')], + imports: [AnalyticsModule, TypeOrmModule.forFeature([Workspace], 'core')], providers: [DomainManagerService, CustomDomainService], exports: [DomainManagerService, CustomDomainService], controllers: [CloudflareController], diff --git a/packages/twenty-server/src/engine/core-modules/domain-manager/services/custom-domain.service.spec.ts b/packages/twenty-server/src/engine/core-modules/domain-manager/services/custom-domain.service.spec.ts index dbfaaa9bc..813e00545 100644 --- a/packages/twenty-server/src/engine/core-modules/domain-manager/services/custom-domain.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/domain-manager/services/custom-domain.service.spec.ts @@ -2,10 +2,12 @@ import { Test, TestingModule } from '@nestjs/testing'; import Cloudflare from 'cloudflare'; import { CustomHostnameCreateResponse } from 'cloudflare/resources/custom-hostnames/custom-hostnames'; +import { AnalyticsContextMock } from 'test/utils/analytics-context.mock'; -import { DomainManagerException } from 'src/engine/core-modules/domain-manager/domain-manager.exception'; import { CustomDomainService } from 'src/engine/core-modules/domain-manager/services/custom-domain.service'; import { DomainManagerService } from 'src/engine/core-modules/domain-manager/services/domain-manager.service'; +import { DomainManagerException } from 'src/engine/core-modules/domain-manager/domain-manager.exception'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; jest.mock('cloudflare'); @@ -25,6 +27,12 @@ describe('CustomDomainService', () => { get: jest.fn(), }, }, + { + provide: AnalyticsService, + useValue: { + createAnalyticsContext: AnalyticsContextMock, + }, + }, { provide: DomainManagerService, useValue: { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index 8824f19e9..7a5bd6966 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -463,6 +463,18 @@ export class ConfigVariables { @IsBoolean() ANALYTICS_ENABLED = false; + @ConfigVariablesMetadata({ + group: ConfigVariablesGroup.AnalyticsConfig, + description: 'Clickhouse host for analytics', + }) + @IsOptional() + @IsUrl({ + require_tld: false, + allow_underscores: true, + }) + @ValidateIf((env) => env.ANALYTICS_ENABLED === true) + CLICKHOUSE_URL: string; + @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Enable or disable telemetry logging', diff --git a/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.spec.ts b/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.spec.ts index c8ba4be29..fa9b04cb6 100644 --- a/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/user-workspace/user-workspace.service.spec.ts @@ -5,7 +5,6 @@ import { Repository } from 'typeorm'; import { TypeORMService } from 'src/database/typeorm/typeorm.service'; import { DatabaseEventAction } from 'src/engine/api/graphql/graphql-query-runner/enums/database-event-action'; -import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants'; import { AuthException } from 'src/engine/core-modules/auth/auth.exception'; import { DomainManagerService } from 'src/engine/core-modules/domain-manager/services/domain-manager.service'; import { UserWorkspace } from 'src/engine/core-modules/user-workspace/user-workspace.entity'; @@ -21,6 +20,7 @@ import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global. import { WorkspaceEventEmitter } from 'src/engine/workspace-event-emitter/workspace-event-emitter'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity'; +import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants'; describe('UserWorkspaceService', () => { let service: UserWorkspaceService; diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts index 3a07d5190..c43ad597f 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts @@ -20,7 +20,7 @@ import { In, Repository } from 'typeorm'; import { FileFolder } from 'src/engine/core-modules/file/interfaces/file-folder.interface'; import { SupportDriver } from 'src/engine/core-modules/twenty-config/interfaces/support.interface'; -import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; import { AuthException, AuthExceptionCode, diff --git a/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts b/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts index b1ce5d021..99ea85204 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/__tests__/workspace.service.spec.ts @@ -25,6 +25,7 @@ import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; describe('WorkspaceService', () => { let service: WorkspaceService; @@ -74,8 +75,13 @@ describe('WorkspaceService', () => { deleteSubscriptions: jest.fn(), }, }, + { + provide: AnalyticsService, + useValue: { + createAnalyticsContext: jest.fn(), + }, + }, ...[ - WorkspaceManagerService, WorkspaceManagerService, UserWorkspaceService, UserService, diff --git a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts index 84b1dc9e3..e7beadd34 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts @@ -44,6 +44,9 @@ import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage import { WorkspaceManagerService } from 'src/engine/workspace-manager/workspace-manager.service'; import { DEFAULT_FEATURE_FLAGS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/default-feature-flags'; import { extractVersionMajorMinorPatch } from 'src/utils/version/extract-version-major-minor-patch'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; +import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-activated'; +import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/custom-domain/custom-domain-deactivated'; @Injectable() // eslint-disable-next-line @nx/workspace-inject-workspace-repository @@ -67,6 +70,7 @@ export class WorkspaceService extends TypeOrmQueryService { private readonly domainManagerService: DomainManagerService, private readonly exceptionHandlerService: ExceptionHandlerService, private readonly permissionsService: PermissionsService, + private readonly analyticsService: AnalyticsService, private readonly customDomainService: CustomDomainService, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, @InjectMessageQueue(MessageQueue.deleteCascadeQueue) @@ -413,6 +417,17 @@ export class WorkspaceService extends TypeOrmQueryService { if (workspace.isCustomDomainEnabled !== isCustomDomainWorking) { workspace.isCustomDomainEnabled = isCustomDomainWorking; await this.workspaceRepository.save(workspace); + + const analytics = this.analyticsService.createAnalyticsContext({ + workspaceId: workspace.id, + }); + + analytics.track( + workspace.isCustomDomainEnabled + ? CUSTOM_DOMAIN_ACTIVATED_EVENT + : CUSTOM_DOMAIN_DEACTIVATED_EVENT, + {}, + ); } return customDomainDetails; diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index 138677c50..e9b5f562a 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -24,6 +24,7 @@ import { RoleModule } from 'src/engine/metadata-modules/role/role.module'; import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; +import { AnalyticsModule } from 'src/engine/core-modules/analytics/analytics.module'; import { workspaceAutoResolverOpts } from './workspace.auto-resolver-opts'; import { Workspace } from './workspace.entity'; @@ -54,6 +55,7 @@ import { WorkspaceService } from './services/workspace.service'; TypeORMModule, PermissionsModule, WorkspaceCacheStorageModule, + AnalyticsModule, RoleModule, ], services: [WorkspaceService], diff --git a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts index 058bd2277..2dae70a4a 100644 --- a/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts @@ -10,7 +10,7 @@ import { IsNull, Not, Repository } from 'typeorm'; import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception'; import { ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface'; -import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service'; import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content'; import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator'; @@ -35,6 +35,7 @@ import { ServerlessFunctionException, ServerlessFunctionExceptionCode, } from 'src/engine/metadata-modules/serverless-function/serverless-function.exception'; +import { SERVERLESS_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/serverless-function/serverless-function-executed'; @Injectable() export class ServerlessFunctionService { @@ -143,9 +144,11 @@ export class ServerlessFunctionService { version, ); - const eventInput = { - action: 'serverlessFunction.executed', - payload: { + this.analyticsService + .createAnalyticsContext({ + workspaceId, + }) + .track(SERVERLESS_FUNCTION_EXECUTED_EVENT, { duration: resultServerlessFunction.duration, status: resultServerlessFunction.status, ...(resultServerlessFunction.error && { @@ -153,14 +156,7 @@ export class ServerlessFunctionService { }), functionId: functionToExecute.id, functionName: functionToExecute.name, - }, - }; - - this.analyticsService.create( - eventInput, - 'serverless-function', - workspaceId, - ); + }); return resultServerlessFunction; } diff --git a/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-telemetry.service.ts b/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-telemetry.service.ts index 720073490..819bc0fbe 100644 --- a/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-telemetry.service.ts +++ b/packages/twenty-server/src/modules/messaging/monitoring/services/messaging-telemetry.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; -import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service'; -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; +import { MONITORING_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/monitoring/monitoring'; type MessagingTelemetryTrackInput = { eventName: string; @@ -14,10 +14,7 @@ type MessagingTelemetryTrackInput = { @Injectable() export class MessagingTelemetryService { - constructor( - private readonly analyticsService: AnalyticsService, - private readonly twentyConfigService: TwentyConfigService, - ) {} + constructor(private readonly analyticsService: AnalyticsService) {} public async track({ eventName, @@ -27,20 +24,16 @@ export class MessagingTelemetryService { messageChannelId, message, }: MessagingTelemetryTrackInput): Promise { - await this.analyticsService.create( - { - action: 'monitoring', - payload: { - eventName: `messaging.${eventName}`, - workspaceId, - userId, - connectedAccountId, - messageChannelId, - message, - }, - }, - userId, - workspaceId, - ); + await this.analyticsService + .createAnalyticsContext({ + userId, + workspaceId, + }) + .track(MONITORING_EVENT, { + eventName: `messaging.${eventName}`, + connectedAccountId, + messageChannelId, + message, + }); } } diff --git a/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts b/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts index 7d6c8c04f..f39703014 100644 --- a/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts +++ b/packages/twenty-server/src/modules/timeline/jobs/create-audit-log-from-internal-event.ts @@ -8,6 +8,10 @@ import { AuditLogRepository } from 'src/modules/timeline/repositiories/audit-log import { AuditLogWorkspaceEntity } from 'src/modules/timeline/standard-objects/audit-log.workspace-entity'; import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; +import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-updated'; +import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created'; +import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-delete'; @Processor(MessageQueue.entityEventsToDbQueue) export class CreateAuditLogFromInternalEvent { @@ -16,6 +20,7 @@ export class CreateAuditLogFromInternalEvent { private readonly workspaceMemberService: WorkspaceMemberRepository, @InjectObjectMetadataRepository(AuditLogWorkspaceEntity) private readonly auditLogRepository: AuditLogRepository, + private readonly analyticsService: AnalyticsService, ) {} @Process(CreateAuditLogFromInternalEvent.name) @@ -48,6 +53,19 @@ export class CreateAuditLogFromInternalEvent { eventData.recordId, workspaceEventBatch.workspaceId, ); + + const analytics = this.analyticsService.createAnalyticsContext({ + workspaceId: workspaceEventBatch.workspaceId, + userId: eventData.userId, + }); + + if (workspaceEventBatch.name.endsWith('.updated')) { + analytics.track(OBJECT_RECORD_UPDATED_EVENT, eventData.properties); + } else if (workspaceEventBatch.name.endsWith('.created')) { + analytics.track(OBJECT_RECORD_CREATED_EVENT, eventData.properties); + } else if (workspaceEventBatch.name.endsWith('.deleted')) { + analytics.track(OBJECT_RECORD_DELETED_EVENT, eventData.properties); + } } } } diff --git a/packages/twenty-server/src/modules/timeline/jobs/timeline-job.module.ts b/packages/twenty-server/src/modules/timeline/jobs/timeline-job.module.ts index 416e81aed..d4480a53c 100644 --- a/packages/twenty-server/src/modules/timeline/jobs/timeline-job.module.ts +++ b/packages/twenty-server/src/modules/timeline/jobs/timeline-job.module.ts @@ -6,6 +6,7 @@ import { UpsertTimelineActivityFromInternalEvent } from 'src/modules/timeline/jo import { AuditLogWorkspaceEntity } from 'src/modules/timeline/standard-objects/audit-log.workspace-entity'; import { TimelineActivityModule } from 'src/modules/timeline/timeline-activity.module'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; +import { AnalyticsModule } from 'src/engine/core-modules/analytics/analytics.module'; @Module({ imports: [ @@ -14,6 +15,7 @@ import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/sta AuditLogWorkspaceEntity, ]), TimelineActivityModule, + AnalyticsModule, ], providers: [ CreateAuditLogFromInternalEvent, diff --git a/packages/twenty-server/src/modules/webhook/jobs/call-webhook.job.ts b/packages/twenty-server/src/modules/webhook/jobs/call-webhook.job.ts index 74ce5f423..d0fb5d31f 100644 --- a/packages/twenty-server/src/modules/webhook/jobs/call-webhook.job.ts +++ b/packages/twenty-server/src/modules/webhook/jobs/call-webhook.job.ts @@ -3,10 +3,11 @@ import { Logger } from '@nestjs/common'; import crypto from 'crypto'; -import { AnalyticsService } from 'src/engine/core-modules/analytics/analytics.service'; +import { AnalyticsService } from 'src/engine/core-modules/analytics/services/analytics.service'; import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator'; import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator'; import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants'; +import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/webhook/webhook-response'; export type CallWebhookJobData = { targetUrl: string; @@ -46,6 +47,9 @@ export class CallWebhookJob { webhookId: data.webhookId, eventName: data.eventName, }; + const analytics = this.analyticsService.createAnalyticsContext({ + workspaceId: data.workspaceId, + }); try { const headers: Record = { @@ -73,27 +77,18 @@ export class CallWebhookJob { ); const success = response.status >= 200 && response.status < 300; - const eventInput = { - action: 'webhook.response', - payload: { - status: response.status, - success, - ...commonPayload, - }, - }; - this.analyticsService.create(eventInput, 'webhook', data.workspaceId); + analytics.track(WEBHOOK_RESPONSE_EVENT, { + status: response.status, + success, + ...commonPayload, + }); } catch (err) { - const eventInput = { - action: 'webhook.response', - payload: { - success: false, - ...commonPayload, - ...(err.response && { status: err.response.status }), - }, - }; - - this.analyticsService.create(eventInput, 'webhook', data.workspaceId); + analytics.track(WEBHOOK_RESPONSE_EVENT, { + success: false, + ...commonPayload, + ...(err.response && { status: err.response.status }), + }); this.logger.error( `Error calling webhook on targetUrl '${data.targetUrl}': ${err}`, ); diff --git a/packages/twenty-server/test/integration/analytics/suites/clickhouse-event-registration.integration-spec.ts b/packages/twenty-server/test/integration/analytics/suites/clickhouse-event-registration.integration-spec.ts new file mode 100644 index 000000000..e8924ef91 --- /dev/null +++ b/packages/twenty-server/test/integration/analytics/suites/clickhouse-event-registration.integration-spec.ts @@ -0,0 +1,74 @@ +import process from 'process'; + +import request from 'supertest'; +import { createClient, ClickHouseClient } from '@clickhouse/client'; + +import { GenericTrackEvent } from 'src/engine/core-modules/analytics/utils/events/track/track'; +import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/analytics/utils/events/track/object-record/object-record-created'; +describe('ClickHouse Event Registration (integration)', () => { + let clickhouseClient: ClickHouseClient; + + beforeAll(async () => { + jest.useRealTimers(); + + clickhouseClient = createClient({ + url: process.env.CLICKHOUSE_URL, + }); + + await clickhouseClient.query({ + query: 'TRUNCATE TABLE events', + format: 'JSONEachRow', + }); + }); + + afterAll(async () => { + if (clickhouseClient) { + await clickhouseClient.close(); + } + }); + + it('should register events in ClickHouse when sending an event', async () => { + const mutation = ` + mutation TrackAnalytics($type: AnalyticsType!, $event: String, $name: String, $properties: JSON) { + trackAnalytics(type: $type, event: $event, name: $name, properties: $properties) { + success + } + } + `; + + const variables = { + type: 'TRACK', + event: OBJECT_RECORD_CREATED_EVENT, + properties: {}, + }; + + const response = await request(`http://localhost:${APP_PORT}`) + .post('/graphql') + .send({ + query: mutation, + variables, + }); + + expect(response.status).toBe(200); + expect(response.body.data.trackAnalytics.success).toBe(true); + + const queryResult = await clickhouseClient.query({ + query: ` + SELECT * + FROM events + WHERE event = '${OBJECT_RECORD_CREATED_EVENT}' AND timestamp >= now() - INTERVAL 1 SECOND + + `, + format: 'JSONEachRow', + }); + + const rows = await queryResult.json(); + + expect(rows.length).toEqual(1); + expect(rows[0].properties).toEqual(variables.properties); + expect(rows[0].event).toEqual(variables.event); + expect(rows[0].workspaceId).toEqual(''); + expect(rows[0].userId).toEqual(''); + expect(rows[0].timestamp).toHaveLength(23); + }); +}); diff --git a/packages/twenty-server/test/utils/analytics-context.mock.ts b/packages/twenty-server/test/utils/analytics-context.mock.ts new file mode 100644 index 000000000..d319a1c59 --- /dev/null +++ b/packages/twenty-server/test/utils/analytics-context.mock.ts @@ -0,0 +1,19 @@ +import { TrackEventName } from 'src/engine/core-modules/analytics/types/events.type'; + +export const AnalyticsContextMock = (params?: { + track?: + | (( + event: TrackEventName, + properties: any, + ) => Promise<{ success: boolean }>) + | jest.Mock; + pageview?: + | ((name: string, properties: any) => Promise<{ success: boolean }>) + | jest.Mock; +}) => { + return { + track: params?.track ?? jest.fn().mockResolvedValue({ success: true }), + pageview: + params?.pageview ?? jest.fn().mockResolvedValue({ success: true }), + }; +}; diff --git a/yarn.lock b/yarn.lock index a0a0282d6..0beeaeb53 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5194,6 +5194,22 @@ __metadata: languageName: node linkType: hard +"@clickhouse/client-common@npm:1.11.0": + version: 1.11.0 + resolution: "@clickhouse/client-common@npm:1.11.0" + checksum: 10c0/84a74458acdf279bc4e844c235d0f658cb77f82e90f19fa285481c31f2f17210a5e801a61f080d6dfd100d970bc4e15a5692d2b7fd067502fbaad99d0dc51310 + languageName: node + linkType: hard + +"@clickhouse/client@npm:^1.11.0": + version: 1.11.0 + resolution: "@clickhouse/client@npm:1.11.0" + dependencies: + "@clickhouse/client-common": "npm:1.11.0" + checksum: 10c0/8557de33dd9c88d2813d563f89b49e666dc2d8166c9131e9727f6935db8dc65428927b4b4ab20b1bc7646b9c38c1eb8d1e372ba483b0b2bb69d64ed89488dec2 + languageName: node + linkType: hard + "@codemirror/autocomplete@npm:^6.0.0, @codemirror/autocomplete@npm:^6.4.0": version: 6.18.0 resolution: "@codemirror/autocomplete@npm:6.18.0" @@ -54989,6 +55005,7 @@ __metadata: version: 0.0.0-use.local resolution: "twenty-server@workspace:packages/twenty-server" dependencies: + "@clickhouse/client": "npm:^1.11.0" "@esbuild-plugins/node-modules-polyfill": "npm:^0.2.2" "@graphql-yoga/nestjs": "patch:@graphql-yoga/nestjs@2.1.0#./patches/@graphql-yoga+nestjs+2.1.0.patch" "@langchain/mistralai": "npm:^0.0.24"