New Timeline (#4936)

Refactored the code to introduce two different concepts:
- AuditLogs (immutable, raw data)
- TimelineActivities (user-friendly, transformed data)

Still some work needed:
- Add message, files, calendar events to timeline (~2 hours if done
naively)
- Refactor repository to try to abstract concept when we can (tbd, wait
for Twenty ORM)
- Introduce ability to display child timelines on parent timeline with
filtering (~2 days)
- Improve UI: add links to open note/task, improve diff display, etc
(half a day)
- Decide the path forward for Task vs Notes: either introduce a new
field type "Record Type" and start going into that direction ; or split
in two objects?
- Trigger updates when a field is changed (will be solved by real-time /
websockets: 2 weeks)
- Integrate behavioral events (1 day for POC, 1 week for
clean/documented)

<img width="1248" alt="Screenshot 2024-04-12 at 09 24 49"
src="https://github.com/twentyhq/twenty/assets/6399865/9428db1a-ab2b-492c-8b0b-d4d9a36e81fa">

packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-create.event.ts

@@ -1,7 +1,7 @@
 import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event';
 
 export class ObjectRecordCreateEvent<T> extends ObjectRecordBaseEvent {
-  details: {
+  properties: {
     after: T;
   };
 }

packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-delete.event.ts

@@ -1,7 +1,7 @@
 import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event';
 
 export class ObjectRecordDeleteEvent<T> extends ObjectRecordBaseEvent {
-  details: {
+  properties: {
     before: T;
   };
 }

packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-job-data.ts

@@ -0,0 +1,11 @@
+import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event';
+
+export class ObjectRecordJobData extends ObjectRecordBaseEvent {
+  getOperation() {
+    return this.name.split('.')[1];
+  }
+
+  getObjectName() {
+    return this.name.split('.')[0];
+  }
+}

packages/twenty-server/src/engine/integrations/event-emitter/types/object-record-update.event.ts

@@ -1,7 +1,7 @@
 import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event';
 
 export class ObjectRecordUpdateEvent<T> extends ObjectRecordBaseEvent {
-  details: {
+  properties: {
     before: T;
     after: T;
     diff?: Partial<T>;

packages/twenty-server/src/engine/integrations/event-emitter/types/object-record.base.event.ts

@@ -1,9 +1,11 @@
 import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
 
 export class ObjectRecordBaseEvent {
+  name: string;
   workspaceId: string;
   recordId: string;
   userId?: string;
+  workspaceMemberId?: string;
   objectMetadata: ObjectMetadataInterface;
-  details: any;
+  properties: any;
 }

packages/twenty-server/src/engine/integrations/event-emitter/utils/__tests__/object-record-changed-values.spec.ts

@@ -1,11 +1,35 @@
+import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
+
 import { objectRecordChangedValues } from 'src/engine/integrations/event-emitter/utils/object-record-changed-values';
 
+const mockObjectMetadata: ObjectMetadataInterface = {
+  id: '1',
+  nameSingular: 'Object',
+  namePlural: 'Objects',
+  labelSingular: 'Object',
+  labelPlural: 'Objects',
+  description: 'Test object metadata',
+  targetTableName: 'test_table',
+  fromRelations: [],
+  toRelations: [],
+  fields: [],
+  isSystem: false,
+  isCustom: false,
+  isActive: true,
+  isRemote: false,
+  isAuditLogged: true,
+};
+
 describe('objectRecordChangedValues', () => {
   it('detects changes in scalar values correctly', () => {
     const oldRecord = { id: 1, name: 'Original Name', updatedAt: new Date() };
     const newRecord = { id: 1, name: 'Updated Name', updatedAt: new Date() };
 
-    const result = objectRecordChangedValues(oldRecord, newRecord);
+    const result = objectRecordChangedValues(
+      oldRecord,
+      newRecord,
+      mockObjectMetadata,
+    );
 
     expect(result).toEqual({
       name: { before: 'Original Name', after: 'Updated Name' },
@@ -13,20 +37,15 @@ describe('objectRecordChangedValues', () => {
   });
 });
 
-it('ignores changes in properties that are objects', () => {
-  const oldRecord = { id: 1, details: { age: 20 } };
-  const newRecord = { id: 1, details: { age: 21 } };
-
-  const result = objectRecordChangedValues(oldRecord, newRecord);
-
-  expect(result).toEqual({});
-});
-
 it('ignores changes to the updatedAt field', () => {
   const oldRecord = { id: 1, updatedAt: new Date('2020-01-01') };
   const newRecord = { id: 1, updatedAt: new Date('2024-01-01') };
 
-  const result = objectRecordChangedValues(oldRecord, newRecord);
+  const result = objectRecordChangedValues(
+    oldRecord,
+    newRecord,
+    mockObjectMetadata,
+  );
 
   expect(result).toEqual({});
 });
@@ -35,7 +54,11 @@ it('returns an empty object when there are no changes', () => {
   const oldRecord = { id: 1, name: 'Name', value: 100 };
   const newRecord = { id: 1, name: 'Name', value: 100 };
 
-  const result = objectRecordChangedValues(oldRecord, newRecord);
+  const result = objectRecordChangedValues(
+    oldRecord,
+    newRecord,
+    mockObjectMetadata,
+  );
 
   expect(result).toEqual({});
 });
@@ -57,9 +80,14 @@ it('correctly handles a mix of changed, unchanged, and special case values', ()
   };
   const expectedChanges = {
     name: { before: 'Original', after: 'Updated' },
+    config: { before: { theme: 'dark' }, after: { theme: 'light' } },
   };
 
-  const result = objectRecordChangedValues(oldRecord, newRecord);
+  const result = objectRecordChangedValues(
+    oldRecord,
+    newRecord,
+    mockObjectMetadata,
+  );
 
   expect(result).toEqual(expectedChanges);
 });

packages/twenty-server/src/engine/integrations/event-emitter/utils/object-record-changed-values.ts

@@ -1,21 +1,30 @@
 import deepEqual from 'deep-equal';
 
+import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
+
+import { FieldMetadataType } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';
+
 export const objectRecordChangedValues = (
   oldRecord: Record<string, any>,
   newRecord: Record<string, any>,
+  objectMetadata: ObjectMetadataInterface,
 ) => {
-  const isObject = (value: any) => {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-  };
-
   const changedValues = Object.keys(newRecord).reduce(
     (acc, key) => {
-      // Discard if values are objects (e.g. we don't want Company.AccountOwner ; we have AccountOwnerId already)
-      if (isObject(oldRecord[key]) || isObject(newRecord[key])) {
+      if (
+        objectMetadata.fields.find(
+          (field) =>
+            field.type === FieldMetadataType.RELATION && field.name === key,
+        )
+      ) {
         return acc;
       }
 
-      if (!deepEqual(oldRecord[key], newRecord[key]) && key != 'updatedAt') {
+      if (objectMetadata.nameSingular === 'activity' && key === 'body') {
+        return acc;
+      }
+
+      if (!deepEqual(oldRecord[key], newRecord[key]) && key !== 'updatedAt') {
         acc[key] = { before: oldRecord[key], after: newRecord[key] };
       }
 

packages/twenty-server/src/engine/integrations/event-emitter/utils/object-record-diff-merge.ts

@@ -0,0 +1,30 @@
+export function objectRecordDiffMerge(
+  oldRecord: Record<string, any>,
+  newRecord: Record<string, any>,
+): Record<string, any> {
+  const result: Record<string, any> = { diff: {} };
+
+  // Iterate over the keys in the oldRecord diff
+  Object.keys(oldRecord.diff ?? {}).forEach((key) => {
+    if (newRecord.diff && newRecord.diff[key]) {
+      // If the key also exists in the newRecord, merge the 'before' from the oldRecord and the 'after' from the newRecord
+      result.diff[key] = {
+        before: oldRecord.diff[key].before,
+        after: newRecord.diff[key].after,
+      };
+    } else {
+      // If the key does not exist in the newRecord, copy it as is from the oldRecord
+      result.diff[key] = oldRecord.diff[key];
+    }
+  });
+
+  // Iterate over the keys in the newRecord diff to catch any that weren't in the oldRecord
+  Object.keys(newRecord.diff ?? {}).forEach((key) => {
+    if (!result.diff[key]) {
+      // If the key was not already added from the oldRecord, add it from the newRecord
+      result.diff[key] = newRecord.diff[key];
+    }
+  });
+
+  return result;
+}

packages/twenty-server/src/engine/integrations/message-queue/jobs.module.ts

@@ -9,8 +9,15 @@ import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
 import { CallWebhookJobsJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook-jobs.job';
 import { CallWebhookJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/call-webhook.job';
 import { RecordPositionBackfillJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/record-position-backfill.job';
-import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
 import { RecordPositionBackfillModule } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-module';
+import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/modules/calendar/jobs/delete-connected-account-associated-calendar-data.job';
+import { GoogleAPIRefreshAccessTokenModule } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.module';
+import { MessageParticipantModule } from 'src/modules/messaging/services/message-participant/message-participant.module';
+import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
+import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
+import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
+import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
+import { AuditLogObjectMetadata } from 'src/modules/timeline/standard-objects/audit-log.object-metadata';
 import { BillingModule } from 'src/engine/core-modules/billing/billing.module';
 import { UpdateSubscriptionJob } from 'src/engine/core-modules/billing/jobs/update-subscription.job';
 import { StripeModule } from 'src/engine/core-modules/billing/stripe/stripe.module';
@@ -24,24 +31,18 @@ import { EnvironmentModule } from 'src/engine/integrations/environment/environme
 import { DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity';
 import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
 import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
-import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
 import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
 import { CleanInactiveWorkspaceJob } from 'src/engine/workspace-manager/workspace-cleaner/crons/clean-inactive-workspace.job';
 import { MatchParticipantJob } from 'src/modules/calendar-messaging-participant/jobs/match-participant.job';
 import { UnmatchParticipantJob } from 'src/modules/calendar-messaging-participant/jobs/unmatch-participant.job';
 import { GoogleCalendarSyncCronJob } from 'src/modules/calendar/crons/jobs/google-calendar-sync.cron.job';
 import { CalendarCreateCompanyAndContactAfterSyncJob } from 'src/modules/calendar/jobs/calendar-create-company-and-contact-after-sync.job';
-import { DeleteConnectedAccountAssociatedCalendarDataJob } from 'src/modules/calendar/jobs/delete-connected-account-associated-calendar-data.job';
 import { GoogleCalendarSyncJob } from 'src/modules/calendar/jobs/google-calendar-sync.job';
 import { CalendarEventCleanerModule } from 'src/modules/calendar/services/calendar-event-cleaner/calendar-event-cleaner.module';
 import { CalendarEventParticipantModule } from 'src/modules/calendar/services/calendar-event-participant/calendar-event-participant.module';
 import { GoogleCalendarSyncModule } from 'src/modules/calendar/services/google-calendar-sync/google-calendar-sync.module';
 import { WorkspaceGoogleCalendarSyncModule } from 'src/modules/calendar/services/workspace-google-calendar-sync/workspace-google-calendar-sync.module';
 import { AutoCompaniesAndContactsCreationModule } from 'src/modules/connected-account/auto-companies-and-contacts-creation/auto-companies-and-contacts-creation.module';
-import { CreateCompanyAndContactJob } from 'src/modules/connected-account/auto-companies-and-contacts-creation/jobs/create-company-and-contact.job';
-import { GoogleAPIRefreshAccessTokenModule } from 'src/modules/connected-account/services/google-api-refresh-access-token/google-api-refresh-access-token.module';
-import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
-import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
 import { GmailFetchMessagesFromCacheCronJob } from 'src/modules/messaging/crons/jobs/gmail-fetch-messages-from-cache.cron.job';
 import { GmailPartialSyncCronJob } from 'src/modules/messaging/crons/jobs/gmail-partial-sync.cron.job';
 import { DeleteConnectedAccountAssociatedMessagingDataJob } from 'src/modules/messaging/jobs/delete-connected-account-associated-messaging-data.job';
@@ -50,12 +51,13 @@ import { GmailFullSyncJob } from 'src/modules/messaging/jobs/gmail-full-sync.job
 import { GmailPartialSyncJob } from 'src/modules/messaging/jobs/gmail-partial-sync.job';
 import { MessagingCreateCompanyAndContactAfterSyncJob } from 'src/modules/messaging/jobs/messaging-create-company-and-contact-after-sync.job';
 import { GmailFetchMessageContentFromCacheModule } from 'src/modules/messaging/services/gmail-fetch-message-content-from-cache/gmail-fetch-message-content-from-cache.module';
+import { CreateAuditLogFromInternalEvent } from 'src/modules/timeline/jobs/create-audit-log-from-internal-event';
+import { UpsertTimelineActivityFromInternalEvent } from 'src/modules/timeline/jobs/upsert-timeline-activity-from-internal-event.job';
 import { GmailFullSyncModule } from 'src/modules/messaging/services/gmail-full-sync/gmail-full-sync.module';
 import { GmailPartialSyncModule } from 'src/modules/messaging/services/gmail-partial-sync/gmail-partial-sync.module';
-import { MessageParticipantModule } from 'src/modules/messaging/services/message-participant/message-participant.module';
 import { ThreadCleanerModule } from 'src/modules/messaging/services/thread-cleaner/thread-cleaner.module';
+import { TimelineActivityModule } from 'src/modules/timeline/timeline-activity.module';
 import { MessageChannelMessageAssociationObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel-message-association.object-metadata';
-import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
 
 @Module({
   imports: [
@@ -83,13 +85,14 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj
     ObjectMetadataRepositoryModule.forFeature([
       ConnectedAccountObjectMetadata,
       MessageChannelObjectMetadata,
-      EventObjectMetadata,
+      AuditLogObjectMetadata,
       MessageChannelMessageAssociationObjectMetadata,
     ]),
     GmailFullSyncModule,
     GmailFetchMessageContentFromCacheModule,
     GmailPartialSyncModule,
     CalendarEventParticipantModule,
+    TimelineActivityModule,
   ],
   providers: [
     {
@@ -156,8 +159,12 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj
     },
 
     {
-      provide: SaveEventToDbJob.name,
-      useClass: SaveEventToDbJob,
+      provide: CreateAuditLogFromInternalEvent.name,
+      useClass: CreateAuditLogFromInternalEvent,
+    },
+    {
+      provide: UpsertTimelineActivityFromInternalEvent.name,
+      useClass: UpsertTimelineActivityFromInternalEvent,
     },
     {
       provide: GmailFetchMessagesFromCacheCronJob.name,