mukeshs/twenty_crm
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Update clickhouse tables (#11905)

Browse Source 
Following a discussion with @Bonapara - changing the base tables
This commit is contained in:
Félix Malfait
2025-05-07 09:39:18 +02:00
committed by GitHub
parent 8b796647f9
commit 7b78b64bca
38 changed files with 345 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-server/nest-cli.json
View File

@ -27,7 +27,7 @@
"outDir": "dist/assets"
},
{
"include": "**/database/clickhouse/migrations/*.sql",
"include": "**/database/clickHouse/migrations/*.sql",
"outDir": "dist/src"
}
],

2
packages/twenty-server/package.json
View File

@ -11,7 +11,7 @@
"worker:prod": "node dist/src/queue-worker/queue-worker",
"database:init:prod": "npx ts-node ./scripts/setup-db.ts && yarn database:migrate:prod",
"database:migrate:prod": "npx -y typeorm migration:run -d dist/src/database/typeorm/metadata/metadata.datasource && npx -y typeorm migration:run -d dist/src/database/typeorm/core/core.datasource",
"clickhouse:migrate:prod": "node dist/src/database/clickhouse/migrations/run-migrations.js",
"clickhouse:migrate:prod": "node dist/src/database/clickHouse/migrations/run-migrations.js",
"typeorm": "../../node_modules/typeorm/.bin/typeorm"
},
"dependencies": {

10
packages/twenty-server/src/database/clickHouse/migrations/001-create-audit-event-table.sql
View File

@ -1,10 +0,0 @@
CREATE TABLE IF NOT EXISTS auditEvent
(
`event` LowCardinality(String),
`timestamp` DateTime64(3),
`userId` String DEFAULT '',
`workspaceId` String DEFAULT '',
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (event, workspaceId, userId, timestamp);

10
packages/twenty-server/src/database/clickHouse/migrations/001-create-workspace-event-table.sql Normal file
View File

@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS workspaceEvent
(
`event` LowCardinality(String) NOT NULL,
`timestamp` DateTime64(3) NOT NULL,
`userId` String DEFAULT '',
`workspaceId` String NOT NULL,
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (workspaceId, event, userId, timestamp);

10
packages/twenty-server/src/database/clickHouse/migrations/002-create-pageview-table.sql
View File

@ -1,10 +1,10 @@
CREATE TABLE IF NOT EXISTS pageview
(
`name` LowCardinality(String),
`timestamp` DateTime64(3),
`properties` JSON,
`name` LowCardinality(String) NOT NULL,
`timestamp` DateTime64(3) NOT NULL,
`userId` String DEFAULT '',
`workspaceId` String DEFAULT ''
`workspaceId` String DEFAULT '',
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (name, workspaceId, userId, timestamp);
ORDER BY (workspaceId, name, userId, timestamp);

12
packages/twenty-server/src/database/clickHouse/migrations/003-create-external-event-table.sql
View File

@ -1,12 +0,0 @@
CREATE TABLE IF NOT EXISTS externalEvent
(
`event` LowCardinality(String) NOT NULL,
`timestamp` DateTime64(3) NOT NULL,
`userId` String DEFAULT '',
`workspaceId` String NOT NULL,
`objectId` String NOT NULL,
`objectType` LowCardinality(String), -- TBC if it should really be a LowCardinality given custom objects
`properties` JSON
)
ENGINE = MergeTree
ORDER BY (event, workspaceId, userId, timestamp);

13
packages/twenty-server/src/database/clickHouse/migrations/003-create-object-event-table.sql Normal file
View File

@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS objectEvent
(
`event` LowCardinality(String) NOT NULL,
`timestamp` DateTime64(3) NOT NULL,
`userId` String DEFAULT '',
`workspaceId` String NOT NULL,
`recordId` String NOT NULL,
`objectMetadataId` String NOT NULL,
`properties` JSON,
`isCustom` Boolean DEFAULT FALSE,
)
ENGINE = MergeTree
ORDER BY (workspaceId, event, userId, timestamp);

6
packages/twenty-server/src/database/clickHouse/migrations/run-migrations.ts
View File

@ -10,7 +10,7 @@ config({
override: true,
});
const clickhouseUrl = () => {
const clickHouseUrl = () => {
const url = process.env.CLICKHOUSE_URL;
if (url) return url;
@ -21,7 +21,7 @@ const clickhouseUrl = () => {
};
async function ensureDatabaseExists() {
const [url, database] = clickhouseUrl().split(/\/(?=[^/]*$)/);
const [url, database] = clickHouseUrl().split(/\/(?=[^/]*$)/);
const client = createClient({
url,
});
@ -74,7 +74,7 @@ async function runMigrations() {
await ensureDatabaseExists();
const client = createClient({
url: clickhouseUrl(),
url: clickHouseUrl(),
clickhouse_settings: {
allow_experimental_json_type: 1,
},

12
packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts
View File

@ -1,9 +1,9 @@
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-deactivated';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-created';
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-delete';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-updated';
import { GenericTrackEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-created';
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-delete';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-updated';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated';
import { GenericTrackEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const fixtures: Array<GenericTrackEvent> = [
{

2
packages/twenty-server/src/database/clickHouse/seeds/run-seeds.ts
View File

@ -18,7 +18,7 @@ async function seedEvents() {
console.log(`⚡ Seeding ${fixtures.length} events...`);
await client.insert({
table: 'auditEvent',
table: 'workspaceEvent',
values: fixtures,
format: 'JSONEachRow',
});

4
packages/twenty-server/src/engine/api/graphql/workspace-query-runner/listeners/telemetry.listener.ts
View File

@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common';
import { OnCustomBatchEvent } from 'src/engine/api/graphql/graphql-query-runner/decorators/on-custom-batch-event.decorator';
import { USER_SIGNUP_EVENT_NAME } from 'src/engine/api/graphql/workspace-query-runner/constants/user-signup-event-name.constants';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { USER_SIGNUP_EVENT } from 'src/engine/core-modules/audit/utils/events/track/user/user-signup';
import { USER_SIGNUP_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/user/user-signup';
import { ObjectRecordCreateEvent } from 'src/engine/core-modules/event-emitter/types/object-record-create.event';
import { TelemetryService } from 'src/engine/core-modules/telemetry/telemetry.service';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event.type';
@ -26,7 +26,7 @@ export class TelemetryListener {
userId: eventPayload.userId,
workspaceId: payload.workspaceId,
})
.track(USER_SIGNUP_EVENT, {});
.insertWorkspaceEvent(USER_SIGNUP_EVENT, {});
this.telemetryService.create(
{

45
packages/twenty-server/src/engine/core-modules/audit/README.md
View File

@ -6,7 +6,11 @@ This module provides analytics tracking functionality for the Twenty application
### Tracking Events
The `AuditService` provides a `createContext` method that returns an object with a `track` method. The `track` method is used to track events.
The `AuditService` provides a `createContext` method that returns an object with three methods:
- `insertWorkspaceEvent`: For tracking workspace-level events
- `createObjectEvent`: For tracking object-level events that include record and metadata IDs
- `createPageviewEvent`: For tracking page views
```typescript
import { Injectable } from '@nestjs/common';
@ -24,10 +28,22 @@ export class MyService {
userId: 'user-id',
});
// Track an event
// The event name will be autocompleted
// The properties will be type-checked based on the event name
analytics.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
// Track a workspace event
analytics.insertWorkspaceEvent(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
// Track an object event
analytics.createObjectEvent(OBJECT_RECORD_CREATED_EVENT, {
recordId: 'record-id',
objectMetadataId: 'object-metadata-id',
// other properties
});
// Track a pageview
analytics.createPageviewEvent('page-name', {
href: '/path',
locale: 'en-US',
// other properties
});
}
}
```
@ -97,8 +113,9 @@ Creates an analytics context with the given user ID and workspace ID.
Returns an object with the following methods:
- `track<T extends TrackEventName>(event: T, properties: TrackEventProperties<T>)`: Tracks an event with the given name and properties
- `pageview(name: string, properties: Partial<PageviewProperties>)`: Tracks a pageview with the given name and properties
- `insertWorkspaceEvent<T extends TrackEventName>(event: T, properties: TrackEventProperties<T>)`: Tracks a workspace-level event
- `createObjectEvent<T extends TrackEventName>(event: T, properties: TrackEventProperties<T> & { recordId: string; objectMetadataId: string })`: Tracks an object-level event
- `createPageviewEvent(name: string, properties: Partial<PageviewProperties>)`: Tracks a pageview
### Types
@ -128,16 +145,4 @@ This approach makes it easier to add new events without having to modify a compl
#### PageviewProperties
A type that defines the structure of pageview properties:
```typescript
type PageviewProperties = {
href: string;
locale: string;
pathname: string;
referrer: string;
sessionId: string;
timeZone: string;
userAgent: string;
};
```
Properties for pageview events, including href, locale, pathname, referrer, sessionId, timeZone, and userAgent.

80
packages/twenty-server/src/engine/core-modules/audit/audit.resolver.spec.ts
View File

@ -38,11 +38,14 @@ describe('AuditResolver', () => {
});
it('should handle a valid pageview input', async () => {
const mockPageview = jest.fn().mockResolvedValue('Pageview created');
const mockInsertPageviewEvent = jest
.fn()
.mockResolvedValue('Pageview created');
auditService.createContext.mockReturnValue({
pageview: mockPageview,
track: jest.fn(),
createPageviewEvent: mockInsertPageviewEvent,
insertWorkspaceEvent: jest.fn(),
createObjectEvent: jest.fn(),
});
const input = {
@ -60,16 +63,19 @@ describe('AuditResolver', () => {
workspaceId: 'workspace-1',
userId: 'user-1',
});
expect(mockPageview).toHaveBeenCalledWith('Test Page', {});
expect(mockInsertPageviewEvent).toHaveBeenCalledWith('Test Page', {});
expect(result).toBe('Pageview created');
});
it('should handle a valid track input', async () => {
const mockTrack = jest.fn().mockResolvedValue('Track created');
const mockInsertWorkspaceEvent = jest
.fn()
.mockResolvedValue('Track created');
auditService.createContext.mockReturnValue({
track: mockTrack,
pageview: jest.fn(),
insertWorkspaceEvent: mockInsertWorkspaceEvent,
createObjectEvent: jest.fn(),
createPageviewEvent: jest.fn(),
});
const input = {
@ -87,10 +93,54 @@ describe('AuditResolver', () => {
workspaceId: 'workspace-2',
userId: 'user-2',
});
expect(mockTrack).toHaveBeenCalledWith('Custom Domain Activated', {});
expect(mockInsertWorkspaceEvent).toHaveBeenCalledWith(
'Custom Domain Activated',
{},
);
expect(result).toBe('Track created');
});
it('should handle object event creation', async () => {
const mockInsertObjectEvent = jest
.fn()
.mockResolvedValue('Object event created');
auditService.createContext.mockReturnValue({
insertWorkspaceEvent: jest.fn(),
createObjectEvent: mockInsertObjectEvent,
createPageviewEvent: jest.fn(),
});
const input = {
event: 'Object Record Created' as const,
recordId: 'test-record-id',
objectMetadataId: 'test-object-metadata-id',
properties: { additionalData: 'test-data' },
};
const result = await resolver.createObjectEvent(
input,
{ id: 'workspace-3' } as Workspace,
{ id: 'user-3' } as User,
);
expect(auditService.createContext).toHaveBeenCalledWith({
workspaceId: 'workspace-3',
userId: 'user-3',
});
expect(mockInsertObjectEvent).toHaveBeenCalledWith(
'Object Record Created',
{
additionalData: 'test-data',
recordId: 'test-record-id',
objectMetadataId: 'test-object-metadata-id',
isCustom: true,
},
);
expect(result).toBe('Object event created');
});
it('should throw an AuditException for invalid input', async () => {
const invalidInput = { type: 'invalid' };
@ -103,4 +153,18 @@ describe('AuditResolver', () => {
),
);
});
it('should throw an AuditException when workspace is missing for createObjectEvent', async () => {
const input = {
event: 'Object Record Created' as const,
recordId: 'test-record-id',
objectMetadataId: 'test-object-metadata-id',
};
await expect(
resolver.createObjectEvent(input, undefined, undefined),
).rejects.toThrowError(
new AuditException('Missing workspace', AuditExceptionCode.INVALID_INPUT),
);
});
});

36
packages/twenty-server/src/engine/core-modules/audit/audit.resolver.ts
View File

@ -4,6 +4,7 @@ import {
AuditException,
AuditExceptionCode,
} from 'src/engine/core-modules/audit/audit.exception';
import { CreateObjectEventInput } from 'src/engine/core-modules/audit/dtos/create-object-event.input';
import { User } from 'src/engine/core-modules/user/user.entity';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { AuthUser } from 'src/engine/decorators/auth/auth-user.decorator';
@ -22,7 +23,7 @@ export class AuditResolver {
constructor(private readonly auditService: AuditService) {}
// preparing for new name
async auditTrack(
async createPageview(
@Args()
createAnalyticsInput: CreateAnalyticsInputV2,
@AuthWorkspace() workspace: Workspace | undefined,
@ -31,6 +32,33 @@ export class AuditResolver {
return this.trackAnalytics(createAnalyticsInput, workspace, user);
}
@Mutation(() => Analytics)
async createObjectEvent(
@Args()
createObjectEventInput: CreateObjectEventInput,
@AuthWorkspace() workspace: Workspace | undefined,
@AuthUser({ allowUndefined: true }) user: User | undefined,
) {
if (!workspace) {
throw new AuditException(
'Missing workspace',
AuditExceptionCode.INVALID_INPUT,
);
}
const analyticsContext = this.auditService.createContext({
workspaceId: workspace.id,
userId: user?.id,
});
return analyticsContext.createObjectEvent(createObjectEventInput.event, {
...createObjectEventInput.properties,
recordId: createObjectEventInput.recordId,
objectMetadataId: createObjectEventInput.objectMetadataId,
isCustom: true,
});
}
@Mutation(() => Analytics)
async trackAnalytics(
@Args()
@ -44,14 +72,16 @@ export class AuditResolver {
});
if (isPageviewAnalyticsInput(createAnalyticsInput)) {
return analyticsContext.pageview(
return analyticsContext.createPageviewEvent(
createAnalyticsInput.name,
createAnalyticsInput.properties ?? {},
);
}
if (isTrackAnalyticsInput(createAnalyticsInput)) {
return analyticsContext.track(
// For track events, we need to determine if it's a workspace or object event
// Since we don't have recordId and objectMetadataId in the input, we use insertWorkspaceEvent
return analyticsContext.insertWorkspaceEvent(
createAnalyticsInput.event,
createAnalyticsInput.properties ?? {},
);

29
packages/twenty-server/src/engine/core-modules/audit/dtos/create-object-event.input.ts Normal file
View File

@ -0,0 +1,29 @@
import { ArgsType, Field } from '@nestjs/graphql';
import { IsNotEmpty, IsObject, IsOptional, IsString } from 'class-validator';
import GraphQLJSON from 'graphql-type-json';
import { TrackEventName } from 'src/engine/core-modules/audit/types/events.type';
@ArgsType()
export class CreateObjectEventInput {
@Field(() => String)
@IsNotEmpty()
@IsString()
event: TrackEventName;
@Field(() => String)
@IsNotEmpty()
@IsString()
recordId: string;
@Field(() => String)
@IsNotEmpty()
@IsString()
objectMetadataId: string;
@Field(() => GraphQLJSON, { nullable: true })
@IsObject()
@IsOptional()
properties?: Record<string, any>;
}

34
packages/twenty-server/src/engine/core-modules/audit/jobs/create-audit-log-from-internal-event.ts
View File

@ -1,23 +1,16 @@
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-created';
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-delete';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-updated';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-created';
import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-delete';
import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/audit/utils/events/object-event/object-record-updated';
import { ObjectRecordEvent } from 'src/engine/core-modules/event-emitter/types/object-record-event.event';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event.type';
import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository';
import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@Processor(MessageQueue.entityEventsToDbQueue)
export class CreateAuditLogFromInternalEvent {
constructor(
@InjectObjectMetadataRepository(WorkspaceMemberWorkspaceEntity)
private readonly workspaceMemberService: WorkspaceMemberRepository,
private readonly auditService: AuditService,
) {}
constructor(private readonly auditService: AuditService) {}
@Process(CreateAuditLogFromInternalEvent.name)
async handle(
@ -38,12 +31,25 @@ export class CreateAuditLogFromInternalEvent {
userId: eventData.userId,
});
// Since these are object record events, we use createObjectEvent
if (workspaceEventBatch.name.endsWith('.updated')) {
analytics.track(OBJECT_RECORD_UPDATED_EVENT, eventProperties);
analytics.createObjectEvent(OBJECT_RECORD_UPDATED_EVENT, {
...eventProperties,
recordId: eventData.recordId,
objectMetadataId: eventData.objectMetadata.id,
});
} else if (workspaceEventBatch.name.endsWith('.created')) {
analytics.track(OBJECT_RECORD_CREATED_EVENT, eventProperties);
analytics.createObjectEvent(OBJECT_RECORD_CREATED_EVENT, {
...eventProperties,
recordId: eventData.recordId,
objectMetadataId: eventData.objectMetadata.id,
});
} else if (workspaceEventBatch.name.endsWith('.deleted')) {
analytics.track(OBJECT_RECORD_DELETED_EVENT, eventProperties);
analytics.createObjectEvent(OBJECT_RECORD_DELETED_EVENT, {
...eventProperties,
recordId: eventData.recordId,
objectMetadataId: eventData.objectMetadata.id,
});
}
}
}

59
packages/twenty-server/src/engine/core-modules/audit/services/audit.service.spec.ts
View File

@ -3,7 +3,7 @@ import { Test, TestingModule } from '@nestjs/testing';
import { AuditContextMock } from 'test/utils/audit-context.mock';
import { ClickHouseService } from 'src/database/clickHouse/clickHouse.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
@ -58,29 +58,37 @@ describe('AuditService', () => {
it('should create a valid context object', () => {
const context = service.createContext(mockUserIdAndWorkspaceId);
expect(context).toHaveProperty('track');
expect(context).toHaveProperty('pageview');
expect(context).toHaveProperty('insertWorkspaceEvent');
expect(context).toHaveProperty('createObjectEvent');
expect(context).toHaveProperty('createPageviewEvent');
});
it('should call track with correct parameters', async () => {
const trackSpy = jest.fn().mockResolvedValue({ success: true });
it('should call insertWorkspaceEvent with correct parameters', async () => {
const insertWorkspaceEventSpy = jest
.fn()
.mockResolvedValue({ success: true });
const mockContext = AuditContextMock({
track: trackSpy,
insertWorkspaceEvent: insertWorkspaceEventSpy,
});
jest.spyOn(service, 'createContext').mockReturnValue(mockContext);
const context = service.createContext(mockUserIdAndWorkspaceId);
await context.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
await context.insertWorkspaceEvent(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
expect(trackSpy).toHaveBeenCalledWith(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
expect(insertWorkspaceEventSpy).toHaveBeenCalledWith(
CUSTOM_DOMAIN_ACTIVATED_EVENT,
{},
);
});
it('should call pageview with correct parameters', async () => {
const pageviewSpy = jest.fn().mockResolvedValue({ success: true });
it('should call createPageviewEvent with correct parameters', async () => {
const createPageviewEventSpy = jest
.fn()
.mockResolvedValue({ success: true });
const mockContext = AuditContextMock({
pageview: pageviewSpy,
createPageviewEvent: createPageviewEventSpy,
});
jest.spyOn(service, 'createContext').mockReturnValue(mockContext);
@ -96,26 +104,43 @@ describe('AuditService', () => {
userAgent: '',
};
await context.pageview('page-view', testPageviewProperties);
await context.createPageviewEvent('page-view', testPageviewProperties);
expect(pageviewSpy).toHaveBeenCalledWith(
expect(createPageviewEventSpy).toHaveBeenCalledWith(
'page-view',
testPageviewProperties,
);
});
it('should return success when track is called', async () => {
it('should return success when insertWorkspaceEvent is called', async () => {
const context = service.createContext(mockUserIdAndWorkspaceId);
const result = await context.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
const result = await context.insertWorkspaceEvent(
CUSTOM_DOMAIN_ACTIVATED_EVENT,
{},
);
expect(result).toEqual({ success: true });
});
it('should return success when pageview is called', async () => {
it('should return success when createPageviewEvent is called', async () => {
const context = service.createContext(mockUserIdAndWorkspaceId);
const result = await context.pageview('page-view', {});
const result = await context.createPageviewEvent('page-view', {});
expect(result).toEqual({ success: true });
});
it('should return success when createObjectEvent is called', async () => {
const context = service.createContext(mockUserIdAndWorkspaceId);
const result = await context.createObjectEvent(
CUSTOM_DOMAIN_ACTIVATED_EVENT,
{
recordId: 'test-record-id',
objectMetadataId: 'test-object-metadata-id',
},
);
expect(result).toEqual({ success: true });
});

21
packages/twenty-server/src/engine/core-modules/audit/services/audit.service.ts
View File

@ -35,16 +35,31 @@ export class AuditService {
: {};
return {
track: <T extends TrackEventName>(
insertWorkspaceEvent: <T extends TrackEventName>(
event: T,
properties: TrackEventProperties<T>,
) =>
this.preventIfDisabled(() =>
this.clickHouseService.insert('auditEvent', [
this.clickHouseService.insert('workspaceEvent', [
{ ...userIdAndWorkspaceId, ...makeTrackEvent(event, properties) },
]),
),
pageview: (name: string, properties: Partial<PageviewProperties>) =>
createObjectEvent: <T extends TrackEventName>(
event: T,
properties: TrackEventProperties<T> & {
recordId: string;
objectMetadataId: string;
},
) =>
this.preventIfDisabled(() =>
this.clickHouseService.insert('objectEvent', [
{ ...userIdAndWorkspaceId, ...makeTrackEvent(event, properties) },
]),
),
createPageviewEvent: (
name: string,
properties: Partial<PageviewProperties>,
) =>
this.preventIfDisabled(() =>
this.clickHouseService.insert('pageview', [
{ ...userIdAndWorkspaceId, ...makePageview(name, properties) },

38
packages/twenty-server/src/engine/core-modules/audit/types/events.type.ts
View File

@ -1,43 +1,43 @@
import {
CUSTOM_DOMAIN_ACTIVATED_EVENT,
CustomDomainActivatedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-activated';
import {
CUSTOM_DOMAIN_DEACTIVATED_EVENT,
CustomDomainDeactivatedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-deactivated';
import {
MONITORING_EVENT,
MonitoringTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/monitoring/monitoring';
import {
OBJECT_RECORD_CREATED_EVENT,
ObjectRecordCreatedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-created';
} from 'src/engine/core-modules/audit/utils/events/object-event/object-record-created';
import {
OBJECT_RECORD_DELETED_EVENT,
ObjectRecordDeletedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-delete';
} from 'src/engine/core-modules/audit/utils/events/object-event/object-record-delete';
import {
OBJECT_RECORD_UPDATED_EVENT,
ObjectRecordUpdatedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-updated';
} from 'src/engine/core-modules/audit/utils/events/object-event/object-record-updated';
import {
CUSTOM_DOMAIN_ACTIVATED_EVENT,
CustomDomainActivatedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import {
CUSTOM_DOMAIN_DEACTIVATED_EVENT,
CustomDomainDeactivatedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated';
import {
MONITORING_EVENT,
MonitoringTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/workspace-event/monitoring/monitoring';
import {
SERVERLESS_FUNCTION_EXECUTED_EVENT,
ServerlessFunctionExecutedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/serverless-function/serverless-function-executed';
} from 'src/engine/core-modules/audit/utils/events/workspace-event/serverless-function/serverless-function-executed';
import {
USER_SIGNUP_EVENT,
UserSignupTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/user/user-signup';
} from 'src/engine/core-modules/audit/utils/events/workspace-event/user/user-signup';
import {
WEBHOOK_RESPONSE_EVENT,
WebhookResponseTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/webhook/webhook-response';
} from 'src/engine/core-modules/audit/utils/events/workspace-event/webhook/webhook-response';
import {
WORKSPACE_ENTITY_CREATED_EVENT,
WorkspaceEntityCreatedTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/workspace-entity/workspace-entity-created';
} from 'src/engine/core-modules/audit/utils/events/workspace-event/workspace-entity/workspace-entity-created';
// Define all track event names
export type TrackEventName =

2
packages/twenty-server/src/engine/core-modules/audit/utils/analytics.utils.ts
View File

@ -12,7 +12,7 @@ import {
import {
eventsRegistry,
GenericTrackEvent,
} from 'src/engine/core-modules/audit/utils/events/track/track';
} from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
const common = (): Record<AuditCommonPropertiesType, string> => ({
timestamp: format(new Date(), 'yyyy-MM-dd HH:mm:ss'),

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/object-record/object-record-created.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/object-event/object-record-created.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const OBJECT_RECORD_CREATED_EVENT = 'Object Record Created' as const;
export const objectRecordCreatedSchema = z.object({

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/object-record/object-record-delete.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/object-event/object-record-delete.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const OBJECT_RECORD_DELETED_EVENT = 'Object Record Deleted' as const;
export const objectRecordDeletedSchema = z.object({

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/object-record/object-record-updated.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/object-event/object-record-updated.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const OBJECT_RECORD_UPDATED_EVENT = 'Object Record Updated' as const;
export const objectRecordUpdatedSchema = z.object({

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-activated.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const CUSTOM_DOMAIN_ACTIVATED_EVENT = 'Custom Domain Activated' as const;
export const customDomainActivatedSchema = z

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-deactivated.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const CUSTOM_DOMAIN_DEACTIVATED_EVENT =
'Custom Domain Deactivated' as const;

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/monitoring/monitoring.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/monitoring/monitoring.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const MONITORING_EVENT = 'Monitoring' as const;
export const monitoringSchema = z

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/serverless-function/serverless-function-executed.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/serverless-function/serverless-function-executed.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const SERVERLESS_FUNCTION_EXECUTED_EVENT =
'Serverless Function Executed' as const;

0
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/track.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/track.ts
View File

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/user/user-signup.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/user/user-signup.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const USER_SIGNUP_EVENT = 'User Signup' as const;
export const userSignupSchema = z

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/webhook/webhook-response.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/webhook/webhook-response.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const WEBHOOK_RESPONSE_EVENT = 'Webhook Response' as const;
export const webhookResponseSchema = z

2
packages/twenty-server/src/engine/core-modules/audit/utils/events/track/workspace-entity/workspace-entity-created.ts → packages/twenty-server/src/engine/core-modules/audit/utils/events/workspace-event/workspace-entity/workspace-entity-created.ts
View File

@ -1,6 +1,6 @@
import { z } from 'zod';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { registerEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
export const WORKSPACE_ENTITY_CREATED_EVENT =
'Workspace Entity Created' as const;

4
packages/twenty-server/src/engine/core-modules/domain-manager/controllers/cloudflare.controller.ts
View File

@ -14,7 +14,7 @@ import { Request, Response } from 'express';
import { Repository } from 'typeorm';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import { AuthRestApiExceptionFilter } from 'src/engine/core-modules/auth/filters/auth-rest-api-exception.filter';
import {
DomainManagerException,
@ -91,7 +91,7 @@ export class CloudflareController {
...workspaceUpdated,
});
await analytics.track(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
await analytics.insertWorkspaceEvent(CUSTOM_DOMAIN_ACTIVATED_EVENT, {});
}
return res.status(200).send();

6
packages/twenty-server/src/engine/core-modules/workspace/services/workspace.service.ts
View File

@ -9,8 +9,8 @@ import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { Repository } from 'typeorm';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-deactivated';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-deactivated';
import { BillingEntitlementKey } from 'src/engine/core-modules/billing/enums/billing-entitlement-key.enum';
import { BillingSubscriptionService } from 'src/engine/core-modules/billing/services/billing-subscription.service';
import { BillingService } from 'src/engine/core-modules/billing/services/billing.service';
@ -422,7 +422,7 @@ export class WorkspaceService extends TypeOrmQueryService<Workspace> {
workspaceId: workspace.id,
});
analytics.track(
analytics.insertWorkspaceEvent(
workspace.isCustomDomainEnabled
? CUSTOM_DOMAIN_ACTIVATED_EVENT
: CUSTOM_DOMAIN_DEACTIVATED_EVENT,

4
packages/twenty-server/src/engine/metadata-modules/serverless-function/serverless-function.service.ts
View File

@ -11,7 +11,7 @@ import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/i
import { ServerlessExecuteResult } from 'src/engine/core-modules/serverless/drivers/interfaces/serverless-driver.interface';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { SERVERLESS_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/serverless-function/serverless-function-executed';
import { SERVERLESS_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/serverless-function/serverless-function-executed';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { readFileContent } from 'src/engine/core-modules/file-storage/utils/read-file-content';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
@ -148,7 +148,7 @@ export class ServerlessFunctionService {
.createContext({
workspaceId,
})
.track(SERVERLESS_FUNCTION_EXECUTED_EVENT, {
.insertWorkspaceEvent(SERVERLESS_FUNCTION_EXECUTED_EVENT, {
duration: resultServerlessFunction.duration,
status: resultServerlessFunction.status,
...(resultServerlessFunction.error && {

2
packages/twenty-server/src/modules/messaging/monitoring/services/messaging-monitoring.service.ts
View File

@ -37,7 +37,7 @@ export class MessagingMonitoringService {
userId,
workspaceId,
})
.track(MONITORING_EVENT, {
.insertWorkspaceEvent(MONITORING_EVENT, {
eventName: `messaging.${eventName}`,
connectedAccountId,
messageChannelId,

6
packages/twenty-server/src/modules/webhook/jobs/call-webhook.job.ts
View File

@ -4,7 +4,7 @@ import { Logger } from '@nestjs/common';
import crypto from 'crypto';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/audit/utils/events/track/webhook/webhook-response';
import { WEBHOOK_RESPONSE_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/webhook/webhook-response';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
@ -78,13 +78,13 @@ export class CallWebhookJob {
const success = response.status >= 200 && response.status < 300;
analytics.track(WEBHOOK_RESPONSE_EVENT, {
analytics.insertWorkspaceEvent(WEBHOOK_RESPONSE_EVENT, {
status: response.status,
success,
...commonPayload,
});
} catch (err) {
analytics.track(WEBHOOK_RESPONSE_EVENT, {
analytics.insertWorkspaceEvent(WEBHOOK_RESPONSE_EVENT, {
success: false,
...commonPayload,
...(err.response && { status: err.response.status }),

12
packages/twenty-server/test/integration/audit/suites/clickHouse-audit-event-registration.integration-spec.ts → packages/twenty-server/test/integration/audit/suites/clickHouse-workspace-event-registration.integration-spec.ts
View File

@ -3,8 +3,8 @@ import process from 'process';
import { ClickHouseClient, createClient } from '@clickhouse/client';
import request from 'supertest';
import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-created';
import { GenericTrackEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/custom-domain/custom-domain-activated';
import { GenericTrackEvent } from 'src/engine/core-modules/audit/utils/events/workspace-event/track';
describe('ClickHouse Event Registration (integration)', () => {
let clickHouseClient: ClickHouseClient;
@ -17,7 +17,7 @@ describe('ClickHouse Event Registration (integration)', () => {
});
await clickHouseClient.query({
query: 'TRUNCATE TABLE auditEvent',
query: 'TRUNCATE TABLE workspaceEvent',
format: 'JSONEachRow',
});
});
@ -39,7 +39,7 @@ describe('ClickHouse Event Registration (integration)', () => {
const variables = {
type: 'TRACK',
event: OBJECT_RECORD_CREATED_EVENT,
event: CUSTOM_DOMAIN_ACTIVATED_EVENT,
properties: {},
};
@ -56,8 +56,8 @@ describe('ClickHouse Event Registration (integration)', () => {
const queryResult = await clickHouseClient.query({
query: `
SELECT *
FROM auditEvent
WHERE event = '${OBJECT_RECORD_CREATED_EVENT}' AND timestamp >= now() - INTERVAL 1 SECOND
FROM workspaceEvent
WHERE event = '${CUSTOM_DOMAIN_ACTIVATED_EVENT}' AND timestamp >= now() - INTERVAL 1 SECOND
`,
format: 'JSONEachRow',

22
packages/twenty-server/test/utils/audit-context.mock.ts
View File

@ -1,19 +1,31 @@
import { TrackEventName } from 'src/engine/core-modules/audit/types/events.type';
export const AuditContextMock = (params?: {
track?:
insertWorkspaceEvent?:
| ((
event: TrackEventName,
properties: any,
) => Promise<{ success: boolean }>)
| jest.Mock<any, any>;
pageview?:
createObjectEvent?:
| ((
event: TrackEventName,
properties: any,
) => Promise<{ success: boolean }>)
| jest.Mock<any, any>;
createPageviewEvent?:
| ((name: string, properties: any) => Promise<{ success: boolean }>)
| jest.Mock<any, any>;
}) => {
return {
track: params?.track ?? jest.fn().mockResolvedValue({ success: true }),
pageview:
params?.pageview ?? jest.fn().mockResolvedValue({ success: true }),
insertWorkspaceEvent:
params?.insertWorkspaceEvent ??
jest.fn().mockResolvedValue({ success: true }),
createObjectEvent:
params?.createObjectEvent ??
jest.fn().mockResolvedValue({ success: true }),
createPageviewEvent:
params?.createPageviewEvent ??
jest.fn().mockResolvedValue({ success: true }),
};
};