Logs show page (#4611)

* Being implementing events on the frontend

* Rename JSON to RAW JSON

* Fix handling of json field on frontend

* Log user id

* Add frontend tests

* Update packages/twenty-server/src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job.ts

Co-authored-by: Weiko <corentin@twenty.com>

* Move db calls to a dedicated repository

* Add server-side tests

---------

Co-authored-by: Weiko <corentin@twenty.com>
This commit is contained in:
Félix Malfait
2024-03-22 14:01:16 +01:00
committed by GitHub
parent aee6d49ea9
commit d876b40056
38 changed files with 488 additions and 95 deletions

View File

@ -2,12 +2,16 @@ import { Injectable } from '@nestjs/common';
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
import { EventRepository } from 'src/modules/event/repositiories/event.repository';
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
import { WorkspaceMemberRepository } from 'src/modules/workspace-member/repositories/workspace-member.repository';
import { WorkspaceMemberObjectMetadata } from 'src/modules/workspace-member/standard-objects/workspace-member.object-metadata';
export type SaveEventToDbJobData = {
workspaceId: string;
recordId: string;
userId: string | undefined;
objectName: string;
operation: string;
details: any;
@ -16,39 +20,47 @@ export type SaveEventToDbJobData = {
@Injectable()
export class SaveEventToDbJob implements MessageQueueJob<SaveEventToDbJobData> {
constructor(
private readonly dataSourceService: DataSourceService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
@InjectObjectMetadataRepository(WorkspaceMemberObjectMetadata)
private readonly workspaceMemberService: WorkspaceMemberRepository,
@InjectObjectMetadataRepository(EventObjectMetadata)
private readonly eventService: EventRepository,
) {}
// TODO: need to support objects others than "person", "company", "opportunity"
async handle(data: SaveEventToDbJobData): Promise<void> {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
data.workspaceId,
);
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
let workspaceMemberId: string | null = null;
if (data.userId) {
const workspaceMember = await this.workspaceMemberService.getByIdOrFail(
data.userId,
data.workspaceId,
);
const eventType = `${data.operation}.${data.objectName}`;
// TODO: add "workspaceMember" (who performed the action, need to send it in the event)
// TODO: need to support objects others than "person", "company", "opportunities"
workspaceMemberId = workspaceMember.id;
}
if (
data.objectName != 'person' &&
data.objectName != 'company' &&
data.objectName != 'opportunities'
data.objectName != 'opportunity'
) {
return;
}
await workspaceDataSource?.query(
`INSERT INTO ${dataSourceMetadata.schema}."event"
("name", "properties", "${data.objectName}Id")
VALUES ('${eventType}', '${JSON.stringify(data.details)}', '${
data.recordId
}') RETURNING *`,
if (data.details.diff) {
// we remove "before" and "after" property for a cleaner/slimmer event payload
data.details = {
diff: data.details.diff,
};
}
await this.eventService.insert(
`${data.operation}.${data.objectName}`,
data.details,
workspaceMemberId,
data.objectName,
data.recordId,
data.workspaceId,
);
}
}

View File

@ -15,6 +15,8 @@ import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { objectRecordChangedValues } from 'src/engine/integrations/event-emitter/utils/object-record-changed-values';
import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event';
@Injectable()
export class EntityEventsToDbListener {
@ -31,7 +33,12 @@ export class EntityEventsToDbListener {
}
@OnEvent('*.updated')
async handleUpdate(payload: ObjectRecordCreateEvent<any>) {
async handleUpdate(payload: ObjectRecordUpdateEvent<any>) {
payload.details.diff = objectRecordChangedValues(
payload.details.before,
payload.details.after,
);
return this.handle(payload, 'updated');
}
@ -58,6 +65,7 @@ export class EntityEventsToDbListener {
this.messageQueueService.add<SaveEventToDbJobData>(SaveEventToDbJob.name, {
workspaceId: payload.workspaceId,
userId: payload.userId,
recordId: payload.recordId,
objectName: payload.objectMetadata.nameSingular,
operation: operation,

View File

@ -9,6 +9,9 @@ import { RecordPositionListener } from 'src/engine/api/graphql/workspace-query-r
import { AuthModule } from 'src/engine/core-modules/auth/auth.module';
import { FeatureFlagEntity } from 'src/engine/core-modules/feature-flag/feature-flag.entity';
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
import { WorkspaceMemberObjectMetadata } from 'src/modules/workspace-member/standard-objects/workspace-member.object-metadata';
import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repository/object-metadata-repository.module';
import { WorkspaceQueryRunnerService } from './workspace-query-runner.service';
@ -21,6 +24,10 @@ import { EntityEventsToDbListener } from './listeners/entity-events-to-db.listen
WorkspaceDataSourceModule,
WorkspacePreQueryHookModule,
TypeOrmModule.forFeature([Workspace, FeatureFlagEntity], 'core'),
ObjectMetadataRepositoryModule.forFeature([
WorkspaceMemberObjectMetadata,
EventObjectMetadata,
]),
],
providers: [
WorkspaceQueryRunnerService,

View File

@ -216,7 +216,7 @@ export class WorkspaceQueryRunnerService {
args: CreateManyResolverArgs<Record>,
options: WorkspaceQueryRunnerOptions,
): Promise<Record[] | undefined> {
const { workspaceId, objectMetadataItem } = options;
const { workspaceId, userId, objectMetadataItem } = options;
const computedArgs = await this.queryRunnerArgsFactory.create(
args,
options,
@ -246,6 +246,7 @@ export class WorkspaceQueryRunnerService {
parsedResults.forEach((record) => {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.created`, {
workspaceId,
userId,
recordId: record.id,
objectMetadata: objectMetadataItem,
details: {
@ -270,7 +271,7 @@ export class WorkspaceQueryRunnerService {
args: UpdateOneResolverArgs<Record>,
options: WorkspaceQueryRunnerOptions,
): Promise<Record | undefined> {
const { workspaceId, objectMetadataItem } = options;
const { workspaceId, userId, objectMetadataItem } = options;
const existingRecord = await this.findOne(
{ filter: { id: { eq: args.id } } } as FindOneResolverArgs,
@ -300,6 +301,7 @@ export class WorkspaceQueryRunnerService {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.updated`, {
workspaceId,
userId,
recordId: (existingRecord as Record).id,
objectMetadata: objectMetadataItem,
details: {
@ -356,7 +358,7 @@ export class WorkspaceQueryRunnerService {
args: DeleteManyResolverArgs<Filter>,
options: WorkspaceQueryRunnerOptions,
): Promise<Record[] | undefined> {
const { workspaceId, objectMetadataItem } = options;
const { workspaceId, userId, objectMetadataItem } = options;
const maximumRecordAffected = this.environmentService.get(
'MUTATION_MAXIMUM_RECORD_AFFECTED',
);
@ -384,6 +386,7 @@ export class WorkspaceQueryRunnerService {
parsedResults.forEach((record) => {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.deleted`, {
workspaceId,
userId,
recordId: record.id,
objectMetadata: objectMetadataItem,
details: {
@ -399,7 +402,7 @@ export class WorkspaceQueryRunnerService {
args: DeleteOneResolverArgs,
options: WorkspaceQueryRunnerOptions,
): Promise<Record | undefined> {
const { workspaceId, objectMetadataItem } = options;
const { workspaceId, userId, objectMetadataItem } = options;
const query = await this.workspaceQueryBuilderFactory.deleteOne(
args,
options,
@ -422,6 +425,7 @@ export class WorkspaceQueryRunnerService {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.deleted`, {
workspaceId,
userId,
recordId: args.id,
objectMetadata: objectMetadataItem,
details: {

View File

@ -8,4 +8,4 @@ export * from './string-filter.input-type';
export * from './time-filter.input-type';
export * from './uuid-filter.input-type';
export * from './boolean-filter.input-type';
export * from './json-filter.input-type';
export * from './raw-json-filter.input-type';

View File

@ -2,8 +2,8 @@ import { GraphQLInputObjectType } from 'graphql';
import { FilterIs } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/filter-is.input-type';
export const JsonFilterType = new GraphQLInputObjectType({
name: 'JsonFilter',
export const RawJsonFilterType = new GraphQLInputObjectType({
name: 'RawJsonFilter',
fields: {
is: { type: FilterIs },
},

View File

@ -32,7 +32,7 @@ import {
IntFilterType,
BooleanFilterType,
BigFloatFilterType,
JsonFilterType,
RawJsonFilterType,
} from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input';
import { OrderByDirectionType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/enum';
import { BigFloatScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@ -70,7 +70,7 @@ export class TypeMapperService {
[FieldMetadataType.PROBABILITY, GraphQLFloat],
[FieldMetadataType.RELATION, GraphQLID],
[FieldMetadataType.POSITION, PositionScalarType],
[FieldMetadataType.JSON, GraphQLJSON],
[FieldMetadataType.RAW_JSON, GraphQLJSON],
]);
return typeScalarMapping.get(fieldMetadataType);
@ -102,7 +102,7 @@ export class TypeMapperService {
[FieldMetadataType.PROBABILITY, FloatFilterType],
[FieldMetadataType.RELATION, UUIDFilterType],
[FieldMetadataType.POSITION, FloatFilterType],
[FieldMetadataType.JSON, JsonFilterType],
[FieldMetadataType.RAW_JSON, RawJsonFilterType],
]);
return typeFilterMapping.get(fieldMetadataType);
@ -126,7 +126,7 @@ export class TypeMapperService {
[FieldMetadataType.SELECT, OrderByDirectionType],
[FieldMetadataType.MULTI_SELECT, OrderByDirectionType],
[FieldMetadataType.POSITION, OrderByDirectionType],
[FieldMetadataType.JSON, OrderByDirectionType],
[FieldMetadataType.RAW_JSON, OrderByDirectionType],
]);
return typeOrderByMapping.get(fieldMetadataType);

View File

@ -69,7 +69,7 @@ const getSchemaComponentsProperties = (
),
};
break;
case FieldMetadataType.JSON:
case FieldMetadataType.RAW_JSON:
type: 'object';
break;
default:

View File

@ -4,5 +4,6 @@ export class ObjectRecordUpdateEvent<T> extends ObjectRecordBaseEvent {
details: {
before: T;
after: T;
diff?: Partial<T>;
};
}

View File

@ -3,6 +3,7 @@ import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metad
export class ObjectRecordBaseEvent {
workspaceId: string;
recordId: string;
userId?: string;
objectMetadata: ObjectMetadataInterface;
details: any;
}

View File

@ -0,0 +1,65 @@
import { objectRecordChangedValues } from 'src/engine/integrations/event-emitter/utils/object-record-changed-values';
describe('objectRecordChangedValues', () => {
it('detects changes in scalar values correctly', () => {
const oldRecord = { id: 1, name: 'Original Name', updatedAt: new Date() };
const newRecord = { id: 1, name: 'Updated Name', updatedAt: new Date() };
const result = objectRecordChangedValues(oldRecord, newRecord);
expect(result).toEqual({
name: { before: 'Original Name', after: 'Updated Name' },
});
});
});
it('ignores changes in properties that are objects', () => {
const oldRecord = { id: 1, details: { age: 20 } };
const newRecord = { id: 1, details: { age: 21 } };
const result = objectRecordChangedValues(oldRecord, newRecord);
expect(result).toEqual({});
});
it('ignores changes to the updatedAt field', () => {
const oldRecord = { id: 1, updatedAt: new Date('2020-01-01') };
const newRecord = { id: 1, updatedAt: new Date('2024-01-01') };
const result = objectRecordChangedValues(oldRecord, newRecord);
expect(result).toEqual({});
});
it('returns an empty object when there are no changes', () => {
const oldRecord = { id: 1, name: 'Name', value: 100 };
const newRecord = { id: 1, name: 'Name', value: 100 };
const result = objectRecordChangedValues(oldRecord, newRecord);
expect(result).toEqual({});
});
it('correctly handles a mix of changed, unchanged, and special case values', () => {
const oldRecord = {
id: 1,
name: 'Original',
status: 'active',
updatedAt: new Date(2020, 1, 1),
config: { theme: 'dark' },
};
const newRecord = {
id: 1,
name: 'Updated',
status: 'active',
updatedAt: new Date(2021, 1, 1),
config: { theme: 'light' },
};
const expectedChanges = {
name: { before: 'Original', after: 'Updated' },
};
const result = objectRecordChangedValues(oldRecord, newRecord);
expect(result).toEqual(expectedChanges);
});

View File

@ -0,0 +1,28 @@
import deepEqual from 'deep-equal';
export const objectRecordChangedValues = (
oldRecord: Record<string, any>,
newRecord: Record<string, any>,
) => {
const isObject = (value: any) => {
return typeof value === 'object' && value !== null && !Array.isArray(value);
};
const changedValues = Object.keys(newRecord).reduce(
(acc, key) => {
// Discard if values are objects (e.g. we don't want Company.AccountOwner ; we have AccountOwnerId already)
if (isObject(oldRecord[key]) || isObject(newRecord[key])) {
return acc;
}
if (!deepEqual(oldRecord[key], newRecord[key]) && key != 'updatedAt') {
acc[key] = { before: oldRecord[key], after: newRecord[key] };
}
return acc;
},
{} as Record<string, { before: any; after: any }>,
);
return changedValues;
};

View File

@ -17,7 +17,7 @@ export class FieldMetadataDefaultValueString {
value: string | null;
}
export class FieldMetadataDefaultValueJson {
export class FieldMetadataDefaultValueRawJson {
@ValidateIf((_object, value) => value !== null)
@IsJSON()
value: JSON | null;

View File

@ -36,7 +36,7 @@ export enum FieldMetadataType {
MULTI_SELECT = 'MULTI_SELECT',
RELATION = 'RELATION',
POSITION = 'POSITION',
JSON = 'JSON',
RAW_JSON = 'RAW_JSON',
}
@Entity('fieldMetadata')

View File

@ -3,7 +3,7 @@ import {
FieldMetadataDefaultValueCurrency,
FieldMetadataDefaultValueDateTime,
FieldMetadataDefaultValueFullName,
FieldMetadataDefaultValueJson,
FieldMetadataDefaultValueRawJson,
FieldMetadataDefaultValueLink,
FieldMetadataDefaultValueNumber,
FieldMetadataDefaultValueString,
@ -51,7 +51,7 @@ type FieldMetadataDefaultValueMapping = {
[FieldMetadataType.RATING]: FieldMetadataDefaultValueString;
[FieldMetadataType.SELECT]: FieldMetadataDefaultValueString;
[FieldMetadataType.MULTI_SELECT]: FieldMetadataDefaultValueStringArray;
[FieldMetadataType.JSON]: FieldMetadataDefaultValueJson;
[FieldMetadataType.RAW_JSON]: FieldMetadataDefaultValueRawJson;
};
type DefaultValueByFieldMetadata<T extends FieldMetadataType | 'default'> = [

View File

@ -35,7 +35,7 @@ export function generateTargetColumnMap(
case FieldMetadataType.SELECT:
case FieldMetadataType.MULTI_SELECT:
case FieldMetadataType.POSITION:
case FieldMetadataType.JSON:
case FieldMetadataType.RAW_JSON:
return {
value: columnName,
};

View File

@ -9,7 +9,7 @@ import {
FieldMetadataDefaultValueCurrency,
FieldMetadataDefaultValueDateTime,
FieldMetadataDefaultValueFullName,
FieldMetadataDefaultValueJson,
FieldMetadataDefaultValueRawJson,
FieldMetadataDefaultValueLink,
FieldMetadataDefaultValueNumber,
FieldMetadataDefaultValueString,
@ -40,7 +40,7 @@ export const defaultValueValidatorsMap = {
[FieldMetadataType.RATING]: [FieldMetadataDefaultValueString],
[FieldMetadataType.SELECT]: [FieldMetadataDefaultValueString],
[FieldMetadataType.MULTI_SELECT]: [FieldMetadataDefaultValueStringArray],
[FieldMetadataType.JSON]: [FieldMetadataDefaultValueJson],
[FieldMetadataType.RAW_JSON]: [FieldMetadataDefaultValueRawJson],
};
export const validateDefaultValueForType = (

View File

@ -29,7 +29,7 @@ export const fieldMetadataTypeToColumnType = <Type extends FieldMetadataType>(
case FieldMetadataType.SELECT:
case FieldMetadataType.MULTI_SELECT:
return 'enum';
case FieldMetadataType.JSON:
case FieldMetadataType.RAW_JSON:
return 'jsonb';
default:
throw new Error(`Cannot convert ${fieldMetadataType} to column type.`);

View File

@ -67,7 +67,7 @@ export class WorkspaceMigrationFactory {
[FieldMetadataType.NUMERIC, { factory: this.basicColumnActionFactory }],
[FieldMetadataType.NUMBER, { factory: this.basicColumnActionFactory }],
[FieldMetadataType.POSITION, { factory: this.basicColumnActionFactory }],
[FieldMetadataType.JSON, { factory: this.basicColumnActionFactory }],
[FieldMetadataType.RAW_JSON, { factory: this.basicColumnActionFactory }],
[
FieldMetadataType.PROBABILITY,
{ factory: this.basicColumnActionFactory },

View File

@ -5,6 +5,7 @@ import { CalendarEventRepository } from 'src/modules/calendar/repositories/calen
import { CompanyRepository } from 'src/modules/company/repositories/company.repository';
import { BlocklistRepository } from 'src/modules/connected-account/repositories/blocklist.repository';
import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository';
import { EventRepository } from 'src/modules/event/repositiories/event.repository';
import { MessageChannelMessageAssociationRepository } from 'src/modules/messaging/repositories/message-channel-message-association.repository';
import { MessageChannelRepository } from 'src/modules/messaging/repositories/message-channel.repository';
import { MessageParticipantRepository } from 'src/modules/messaging/repositories/message-participant.repository';
@ -22,6 +23,7 @@ export const metadataToRepositoryMapping = {
CalendarEventObjectMetadata: CalendarEventRepository,
CompanyObjectMetadata: CompanyRepository,
ConnectedAccountObjectMetadata: ConnectedAccountRepository,
EventObjectMetadata: EventRepository,
MessageChannelMessageAssociationObjectMetadata:
MessageChannelMessageAssociationRepository,
MessageChannelObjectMetadata: MessageChannelRepository,

View File

@ -22,7 +22,7 @@ export const mapFieldMetadataTypeToDataType = (
return 'boolean';
case FieldMetadataType.DATE_TIME:
return 'timestamp';
case FieldMetadataType.JSON:
case FieldMetadataType.RAW_JSON:
return 'jsonb';
case FieldMetadataType.RATING:
case FieldMetadataType.SELECT: