Update what is being audit logged (#11833)

No need to audit log workflow runs as it's already a form of audit log.
Add more audit log for other objects
Rename MessagingTelemetry to MessagingMonitoring
Merge Analytics and Audit in one (Audit)

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

packages/twenty-server/src/database/clickHouse/clickHouse.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+
+import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module';
+
+import { ClickHouseService } from './clickHouse.service';
+
+@Module({
+  imports: [TwentyConfigModule],
+  providers: [ClickHouseService],
+  exports: [ClickHouseService],
+})
+export class ClickHouseModule {}

packages/twenty-server/src/database/clickHouse/clickHouse.service.spec.ts

@@ -0,0 +1,264 @@
+import { Test, TestingModule } from '@nestjs/testing';
+
+import { ClickHouseClient } from '@clickhouse/client';
+
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+
+import { ClickHouseService } from './clickHouse.service';
+
+// Mock the createClient function from @clickhouse/client
+jest.mock('@clickhouse/client', () => ({
+  createClient: jest.fn().mockReturnValue({
+    insert: jest.fn().mockResolvedValue({}),
+    query: jest.fn().mockResolvedValue({
+      json: jest.fn().mockResolvedValue([{ test: 'data' }]),
+    }),
+    ping: jest.fn().mockResolvedValue({ success: true }),
+    close: jest.fn().mockResolvedValue({}),
+    exec: jest.fn().mockResolvedValue({}),
+  }),
+}));
+
+describe('ClickHouseService', () => {
+  let service: ClickHouseService;
+  let twentyConfigService: TwentyConfigService;
+  let mockClickHouseClient: jest.Mocked<ClickHouseClient>;
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+
+    mockClickHouseClient = {
+      insert: jest.fn().mockResolvedValue({}),
+      query: jest.fn().mockResolvedValue({
+        json: jest.fn().mockResolvedValue([{ test: 'data' }]),
+      }),
+      ping: jest.fn().mockResolvedValue({ success: true }),
+      close: jest.fn().mockResolvedValue({}),
+      exec: jest.fn().mockResolvedValue({}),
+    } as unknown as jest.Mocked<ClickHouseClient>;
+
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        ClickHouseService,
+        {
+          provide: TwentyConfigService,
+          useValue: {
+            get: jest.fn((key) => {
+              if (key === 'CLICKHOUSE_URL') return 'http://localhost:8123';
+
+              return undefined;
+            }),
+          },
+        },
+      ],
+    }).compile();
+
+    service = module.get<ClickHouseService>(ClickHouseService);
+    twentyConfigService = module.get<TwentyConfigService>(TwentyConfigService);
+
+    // Set the mock client
+    (service as any).mainClient = mockClickHouseClient;
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+
+  describe('constructor', () => {
+    it('should not initialize clickhouse client when clickhouse is disabled', async () => {
+      jest.spyOn(twentyConfigService, 'get').mockImplementation((key) => {
+        if (key === 'CLICKHOUSE_URL') return '';
+
+        return undefined;
+      });
+
+      const newModule: TestingModule = await Test.createTestingModule({
+        providers: [
+          ClickHouseService,
+          {
+            provide: TwentyConfigService,
+            useValue: twentyConfigService,
+          },
+        ],
+      }).compile();
+
+      const newService = newModule.get<ClickHouseService>(ClickHouseService);
+
+      expect((newService as any).mainClient).toBeUndefined();
+    });
+  });
+
+  describe('insert', () => {
+    it('should insert data into clickhouse and return success', async () => {
+      const testData = [{ id: 1, name: 'test' }];
+      const result = await service.insert('test_table', testData);
+
+      expect(result).toEqual({ success: true });
+      expect(mockClickHouseClient.insert).toHaveBeenCalledWith({
+        table: 'test_table',
+        values: testData,
+        format: 'JSONEachRow',
+      });
+    });
+
+    it('should return failure when clickhouse client is not defined', async () => {
+      (service as any).mainClient = undefined;
+
+      const testData = [{ id: 1, name: 'test' }];
+      const result = await service.insert('test_table', testData);
+
+      expect(result).toEqual({ success: false });
+    });
+
+    it('should handle errors and return failure', async () => {
+      const testError = new Error('Test error');
+
+      mockClickHouseClient.insert.mockRejectedValueOnce(testError);
+
+      const testData = [{ id: 1, name: 'test' }];
+      const result = await service.insert('test_table', testData);
+
+      expect(result).toEqual({ success: false });
+      // Since the service uses logger.error instead of exceptionHandlerService.captureExceptions,
+      // we don't need to assert on exceptionHandlerService
+    });
+  });
+
+  describe('select', () => {
+    it('should execute a query and return results', async () => {
+      const query = 'SELECT * FROM test_table WHERE id = {id:Int32}';
+      const params = { id: 1 };
+
+      mockClickHouseClient.query.mockResolvedValueOnce({
+        json: jest.fn().mockResolvedValueOnce([{ id: 1, name: 'test' }]),
+      } as any);
+
+      const result = await service.select(query, params);
+
+      expect(result).toEqual([{ id: 1, name: 'test' }]);
+      expect(mockClickHouseClient.query).toHaveBeenCalledWith({
+        query,
+        format: 'JSONEachRow',
+        query_params: params,
+      });
+    });
+
+    it('should return empty array when clickhouse client is not defined', async () => {
+      (service as any).mainClient = undefined;
+
+      const query = 'SELECT * FROM test_table';
+      const result = await service.select(query);
+
+      expect(result).toEqual([]);
+    });
+
+    it('should handle errors and return empty array', async () => {
+      const testError = new Error('Test error');
+
+      mockClickHouseClient.query.mockRejectedValueOnce(testError);
+
+      const query = 'SELECT * FROM test_table';
+      const result = await service.select(query);
+
+      expect(result).toEqual([]);
+      // Since the service uses logger.error instead of exceptionHandlerService.captureExceptions,
+      // we don't need to assert on exceptionHandlerService
+    });
+  });
+
+  describe('createDatabase', () => {
+    it('should create a database and return true', async () => {
+      const result = await service.createDatabase('test_db');
+
+      expect(result).toBe(true);
+      expect(mockClickHouseClient.exec).toHaveBeenCalledWith({
+        query: 'CREATE DATABASE IF NOT EXISTS test_db',
+      });
+    });
+
+    it('should return false when clickhouse client is not defined', async () => {
+      (service as any).mainClient = undefined;
+
+      const result = await service.createDatabase('test_db');
+
+      expect(result).toBe(false);
+    });
+  });
+
+  describe('dropDatabase', () => {
+    it('should drop a database and return true', async () => {
+      const result = await service.dropDatabase('test_db');
+
+      expect(result).toBe(true);
+      expect(mockClickHouseClient.exec).toHaveBeenCalledWith({
+        query: 'DROP DATABASE IF EXISTS test_db',
+      });
+    });
+
+    it('should return false when clickhouse client is not defined', async () => {
+      (service as any).mainClient = undefined;
+
+      const result = await service.dropDatabase('test_db');
+
+      expect(result).toBe(false);
+    });
+  });
+
+  describe('connectToClient', () => {
+    it('should connect to a client and return it', async () => {
+      jest
+        .spyOn(service, 'connectToClient')
+        .mockResolvedValueOnce(mockClickHouseClient);
+
+      const result = await service.connectToClient('test-client');
+
+      expect(result).toBe(mockClickHouseClient);
+    });
+
+    it('should reuse an existing client if available', async () => {
+      // Set up a client in the map
+      (service as any).clients.set('test-client', mockClickHouseClient);
+
+      const result = await service.connectToClient('test-client');
+
+      expect(result).toBe(mockClickHouseClient);
+    });
+  });
+
+  describe('disconnectFromClient', () => {
+    it('should disconnect from a client', async () => {
+      // Set up a client in the map
+      (service as any).clients.set('test-client', mockClickHouseClient);
+
+      await service.disconnectFromClient('test-client');
+
+      expect(mockClickHouseClient.close).toHaveBeenCalled();
+      expect((service as any).clients.has('test-client')).toBe(false);
+    });
+
+    it('should do nothing if client does not exist', async () => {
+      await service.disconnectFromClient('non-existent-client');
+
+      expect(mockClickHouseClient.close).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('lifecycle hooks', () => {
+    it('should ping server on module init', async () => {
+      await service.onModuleInit();
+
+      expect(mockClickHouseClient.ping).toHaveBeenCalled();
+    });
+
+    it('should close all clients on module destroy', async () => {
+      // Set up a couple of clients
+      (service as any).clients.set('client1', mockClickHouseClient);
+      (service as any).clients.set('client2', mockClickHouseClient);
+
+      await service.onModuleDestroy();
+
+      // One for mainClient, and two for the clients in the map
+      expect(mockClickHouseClient.close).toHaveBeenCalledTimes(3);
+    });
+  });
+});

packages/twenty-server/src/database/clickHouse/clickHouse.service.ts

@@ -0,0 +1,230 @@
+import {
+  Injectable,
+  Logger,
+  OnModuleDestroy,
+  OnModuleInit,
+} from '@nestjs/common';
+
+import { ClickHouseClient, createClient } from '@clickhouse/client';
+
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+
+@Injectable()
+export class ClickHouseService implements OnModuleInit, OnModuleDestroy {
+  private mainClient: ClickHouseClient | undefined;
+  private clients: Map<string, ClickHouseClient> = new Map();
+  private isClientInitializing: Map<string, boolean> = new Map();
+  private readonly logger = new Logger(ClickHouseService.name);
+
+  constructor(private readonly twentyConfigService: TwentyConfigService) {
+    if (this.twentyConfigService.get('CLICKHOUSE_URL')) {
+      this.mainClient = createClient({
+        url: this.twentyConfigService.get('CLICKHOUSE_URL'),
+        compression: {
+          response: true,
+          request: true,
+        },
+        clickhouse_settings: {
+          async_insert: 1,
+          wait_for_async_insert: 1,
+        },
+        application: 'twenty',
+      });
+    }
+  }
+
+  public getMainClient(): ClickHouseClient | undefined {
+    return this.mainClient;
+  }
+
+  public async connectToClient(
+    clientId: string,
+    url?: string,
+  ): Promise<ClickHouseClient | undefined> {
+    if (!this.twentyConfigService.get('CLICKHOUSE_URL')) {
+      return undefined;
+    }
+
+    // Wait for a bit before trying again if another initialization is in progress
+    while (this.isClientInitializing.get(clientId)) {
+      await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+
+    if (this.clients.has(clientId)) {
+      return this.clients.get(clientId);
+    }
+
+    this.isClientInitializing.set(clientId, true);
+
+    try {
+      const clientInstance = await this.createAndInitializeClient(url);
+
+      this.clients.set(clientId, clientInstance);
+
+      return clientInstance;
+    } catch (err) {
+      this.logger.error(
+        `Error connecting to ClickHouse client ${clientId}`,
+        err,
+      );
+
+      return undefined;
+    } finally {
+      this.isClientInitializing.delete(clientId);
+    }
+  }
+
+  private async createAndInitializeClient(
+    url?: string,
+  ): Promise<ClickHouseClient> {
+    const client = createClient({
+      url: url ?? this.twentyConfigService.get('CLICKHOUSE_URL'),
+      compression: {
+        response: true,
+        request: true,
+      },
+      clickhouse_settings: {
+        async_insert: 1,
+        wait_for_async_insert: 1,
+      },
+      application: 'twenty',
+    });
+
+    // Ping to check connection
+    await client.ping();
+
+    return client;
+  }
+
+  public async disconnectFromClient(clientId: string) {
+    if (!this.clients.has(clientId)) {
+      return;
+    }
+
+    const client = this.clients.get(clientId);
+
+    if (client) {
+      await client.close();
+    }
+
+    this.clients.delete(clientId);
+  }
+
+  async onModuleInit() {
+    if (this.mainClient) {
+      // Just ping to verify the connection
+      try {
+        await this.mainClient.ping();
+      } catch (err) {
+        this.logger.error('Error connecting to ClickHouse', err);
+      }
+    }
+  }
+
+  async onModuleDestroy() {
+    // Close main client
+    if (this.mainClient) {
+      await this.mainClient.close();
+    }
+
+    // Close all other clients
+    for (const [, client] of this.clients) {
+      await client.close();
+    }
+  }
+
+  public async insert<T extends Record<string, any>>(
+    table: string,
+    values: T[],
+    clientId?: string,
+  ): Promise<{ success: boolean }> {
+    try {
+      const client = clientId
+        ? await this.connectToClient(clientId)
+        : this.mainClient;
+
+      if (!client) {
+        return { success: false };
+      }
+
+      await client.insert({
+        table,
+        values,
+        format: 'JSONEachRow',
+      });
+
+      return { success: true };
+    } catch (err) {
+      this.logger.error('Error inserting data into ClickHouse', err);
+
+      return { success: false };
+    }
+  }
+
+  // Method to execute a select query
+  public async select<T>(
+    query: string,
+    params?: Record<string, any>,
+    clientId?: string,
+  ): Promise<T[]> {
+    try {
+      const client = clientId
+        ? await this.connectToClient(clientId)
+        : this.mainClient;
+
+      if (!client) {
+        return [];
+      }
+
+      const resultSet = await client.query({
+        query,
+        format: 'JSONEachRow',
+        query_params: params,
+      });
+
+      const result = await resultSet.json<T>();
+
+      return Array.isArray(result) ? result : [];
+    } catch (err) {
+      this.logger.error('Error executing select query in ClickHouse', err);
+
+      return [];
+    }
+  }
+
+  public async createDatabase(databaseName: string): Promise<boolean> {
+    try {
+      if (!this.mainClient) {
+        return false;
+      }
+
+      await this.mainClient.exec({
+        query: `CREATE DATABASE IF NOT EXISTS ${databaseName}`,
+      });
+
+      return true;
+    } catch (err) {
+      this.logger.error('Error creating database in ClickHouse', err);
+
+      return false;
+    }
+  }
+
+  public async dropDatabase(databaseName: string): Promise<boolean> {
+    try {
+      if (!this.mainClient) {
+        return false;
+      }
+
+      await this.mainClient.exec({
+        query: `DROP DATABASE IF EXISTS ${databaseName}`,
+      });
+
+      return true;
+    } catch (err) {
+      this.logger.error('Error dropping database in ClickHouse', err);
+
+      return false;
+    }
+  }
+}

packages/twenty-server/src/database/clickHouse/migrations/001-create-audit-event-table.sql

@@ -1,4 +1,4 @@
-CREATE TABLE IF NOT EXISTS events
+CREATE TABLE IF NOT EXISTS auditEvent
 (
     `event`      LowCardinality(String),
     `timestamp`   DateTime64(3),
@@ -7,4 +7,4 @@ CREATE TABLE IF NOT EXISTS events
     `properties`  JSON
 )
     ENGINE = MergeTree
-        ORDER BY (event, workspaceId, timestamp);
+        ORDER BY (event, workspaceId, userId, timestamp);

packages/twenty-server/src/database/clickHouse/migrations/003-create-external-event-table.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS externalEvent
+(
+    `event`      LowCardinality(String) NOT NULL,
+    `timestamp`   DateTime64(3) NOT NULL,
+    `userId`      String DEFAULT '',
+    `workspaceId` String NOT NULL,
+    `objectId`    String NOT NULL,
+    `objectType`  LowCardinality(String), -- TBC if it should really be a LowCardinality given custom objects
+    `properties`  JSON
+)
+    ENGINE = MergeTree
+        ORDER BY (event, workspaceId, userId, timestamp);

packages/twenty-server/src/database/clickHouse/migrations/run-migrations.ts

@@ -36,7 +36,7 @@ async function ensureDatabaseExists() {
 async function ensureMigrationTable(client: ClickHouseClient) {
   await client.command({
     query: `
-        CREATE TABLE IF NOT EXISTS migrations (
+        CREATE TABLE IF NOT EXISTS _migration (
           filename String,
           applied_at DateTime DEFAULT now()
         ) ENGINE = MergeTree()
@@ -50,7 +50,7 @@ async function hasMigrationBeenRun(
   client: ClickHouseClient,
 ): Promise<boolean> {
   const resultSet = await client.query({
-    query: `SELECT count() as count FROM migrations WHERE filename = {filename:String}`,
+    query: `SELECT count() as count FROM _migration WHERE filename = {filename:String}`,
     query_params: { filename },
     format: 'JSON',
   });
@@ -61,7 +61,7 @@ async function hasMigrationBeenRun(
 
 async function recordMigration(filename: string, client: ClickHouseClient) {
   await client.insert({
-    table: 'migrations',
+    table: '_migration',
     values: [{ filename }],
     format: 'JSONEachRow',
   });

packages/twenty-server/src/database/clickHouse/seeds/fixtures.ts

@@ -0,0 +1,54 @@
+import { CUSTOM_DOMAIN_ACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-activated';
+import { CUSTOM_DOMAIN_DEACTIVATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/custom-domain/custom-domain-deactivated';
+import { OBJECT_RECORD_CREATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-created';
+import { OBJECT_RECORD_DELETED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-delete';
+import { OBJECT_RECORD_UPDATED_EVENT } from 'src/engine/core-modules/audit/utils/events/track/object-record/object-record-updated';
+import { GenericTrackEvent } from 'src/engine/core-modules/audit/utils/events/track/track';
+
+export const fixtures: Array<GenericTrackEvent> = [
+  {
+    type: 'track',
+    event: CUSTOM_DOMAIN_ACTIVATED_EVENT,
+    timestamp: '2024-10-24T15:55:35.177',
+    version: '1',
+    userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
+    workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
+    properties: {},
+  },
+  {
+    type: 'track',
+    event: CUSTOM_DOMAIN_DEACTIVATED_EVENT,
+    timestamp: '2024-10-24T15:55:35.177',
+    version: '1',
+    userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
+    workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
+    properties: {},
+  },
+  {
+    type: 'track',
+    event: OBJECT_RECORD_CREATED_EVENT,
+    timestamp: '2024-10-24T15:55:35.177',
+    version: '1',
+    userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
+    workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
+    properties: {},
+  },
+  {
+    type: 'track',
+    event: OBJECT_RECORD_UPDATED_EVENT,
+    timestamp: '2024-10-24T15:55:35.177',
+    version: '1',
+    userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
+    workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
+    properties: {},
+  },
+  {
+    type: 'track',
+    event: OBJECT_RECORD_DELETED_EVENT,
+    timestamp: '2024-10-24T15:55:35.177',
+    version: '1',
+    userId: '20202020-9e3b-46d4-a556-88b9ddc2b034',
+    workspaceId: '20202020-1c25-4d02-bf25-6aeccf7ea419',
+    properties: {},
+  },
+];

packages/twenty-server/src/database/clickHouse/seeds/run-seeds.ts

@@ -2,7 +2,7 @@
 import { createClient } from '@clickhouse/client';
 import { config } from 'dotenv';
 
-import { fixtures } from 'src/engine/core-modules/analytics/utils/fixtures/fixtures';
+import { fixtures } from './fixtures';
 
 config({
   path: process.env.NODE_ENV === 'test' ? '.env.test' : '.env',
@@ -18,7 +18,7 @@ async function seedEvents() {
     console.log(`⚡ Seeding ${fixtures.length} events...`);
 
     await client.insert({
-      table: 'events',
+      table: 'auditEvent',
       values: fixtures,
       format: 'JSONEachRow',
     });

packages/twenty-server/src/database/typeorm-seeds/core/feature-flags.ts

@@ -25,11 +25,6 @@ export const seedFeatureFlags = async (
         workspaceId: workspaceId,
         value: true,
       },
-      {
-        key: FeatureFlagKey.IsEventObjectEnabled,
-        workspaceId: workspaceId,
-        value: false,
-      },
       {
         key: FeatureFlagKey.IsStripeIntegrationEnabled,
         workspaceId: workspaceId,
@@ -40,21 +35,11 @@ export const seedFeatureFlags = async (
         workspaceId: workspaceId,
         value: true,
       },
-      {
-        key: FeatureFlagKey.IsAnalyticsV2Enabled,
-        workspaceId: workspaceId,
-        value: true,
-      },
       {
         key: FeatureFlagKey.IsCustomDomainEnabled,
         workspaceId: workspaceId,
         value: false,
       },
-      {
-        key: FeatureFlagKey.IsApprovedAccessDomainsEnabled,
-        workspaceId: workspaceId,
-        value: true,
-      },
       {
         key: FeatureFlagKey.IsUniqueIndexesEnabled,
         workspaceId: workspaceId,