Add JSON field type and Event object (#4566)

* Add JSON field type and Event object

* Simplify code

* Adress PR comments and add featureFlag
This commit is contained in:
Félix Malfait
2024-03-19 21:54:08 +01:00
committed by GitHub
parent 4ab426c52a
commit 4bfb90657f
51 changed files with 575 additions and 117 deletions

View File

@ -0,0 +1,54 @@
import { Injectable } from '@nestjs/common';
import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface';
import { DataSourceService } from 'src/engine-metadata/data-source/data-source.service';
import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
export type SaveEventToDbJobData = {
workspaceId: string;
recordId: string;
objectName: string;
operation: string;
details: any;
};
@Injectable()
export class SaveEventToDbJob implements MessageQueueJob<SaveEventToDbJobData> {
constructor(
private readonly dataSourceService: DataSourceService,
private readonly workspaceDataSourceService: WorkspaceDataSourceService,
) {}
async handle(data: SaveEventToDbJobData): Promise<void> {
const dataSourceMetadata =
await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceIdOrFail(
data.workspaceId,
);
const workspaceDataSource =
await this.workspaceDataSourceService.connectToWorkspaceDataSource(
data.workspaceId,
);
const eventType = `${data.operation}.${data.objectName}`;
// TODO: add "workspaceMember" (who performed the action, need to send it in the event)
// TODO: need to support objects others than "person", "company", "opportunities"
if (
data.objectName != 'person' &&
data.objectName != 'company' &&
data.objectName != 'opportunities'
) {
return;
}
await workspaceDataSource?.query(
`INSERT INTO ${dataSourceMetadata.schema}."event"
("name", "properties", "${data.objectName}Id")
VALUES ('${eventType}', '${JSON.stringify(data.details)}', '${
data.recordId
}') RETURNING *`,
);
}
}

View File

@ -0,0 +1,67 @@
import { Inject, Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import {
SaveEventToDbJobData,
SaveEventToDbJob,
} from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
import {
FeatureFlagEntity,
FeatureFlagKeys,
} from 'src/engine/modules/feature-flag/feature-flag.entity';
@Injectable()
export class EntityEventsToDbListener {
constructor(
@Inject(MessageQueue.entityEventsToDbQueue)
private readonly messageQueueService: MessageQueueService,
@InjectRepository(FeatureFlagEntity, 'core')
private readonly featureFlagRepository: Repository<FeatureFlagEntity>,
) {}
@OnEvent('*.created')
async handleCreate(payload: ObjectRecordCreateEvent<any>) {
return this.handle(payload, 'created');
}
@OnEvent('*.updated')
async handleUpdate(payload: ObjectRecordCreateEvent<any>) {
return this.handle(payload, 'updated');
}
// @OnEvent('*.deleted') - TODO: implement when we have soft deleted
// ....
private async handle(
payload: ObjectRecordCreateEvent<any>,
operation: string,
) {
const isEventObjectEnabledFeatureFlag =
await this.featureFlagRepository.findOneBy({
workspaceId: payload.workspaceId,
key: FeatureFlagKeys.IsEventObjectEnabled,
value: true,
});
if (
!isEventObjectEnabledFeatureFlag ||
!isEventObjectEnabledFeatureFlag.value
) {
return;
}
this.messageQueueService.add<SaveEventToDbJobData>(SaveEventToDbJob.name, {
workspaceId: payload.workspaceId,
recordId: payload.recordId,
objectName: payload.objectMetadata.nameSingular,
operation: operation,
details: payload.details,
});
}
}

View File

@ -1,10 +1,9 @@
import { Inject, Injectable } from '@nestjs/common';
import { OnEvent } from '@nestjs/event-emitter';
import {
CreatedObjectMetadata,
ObjectRecordCreateEvent,
} from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { ObjectMetadataInterface } from 'src/engine-metadata/field-metadata/interfaces/object-metadata.interface';
import { ObjectRecordCreateEvent } from 'src/engine/integrations/event-emitter/types/object-record-create.event';
import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service';
import {
@ -21,11 +20,11 @@ export class RecordPositionListener {
@OnEvent('*.created')
async handleAllCreate(payload: ObjectRecordCreateEvent<any>) {
if (!hasPositionField(payload.createdObjectMetadata)) {
if (!hasPositionField(payload.objectMetadata)) {
return;
}
if (hasPositionSet(payload.createdRecord)) {
if (hasPositionSet(payload.details.after)) {
return;
}
@ -33,15 +32,19 @@ export class RecordPositionListener {
RecordPositionBackfillJob.name,
{
workspaceId: payload.workspaceId,
recordId: payload.createdRecord.id,
objectMetadata: payload.createdObjectMetadata,
recordId: payload.recordId,
objectMetadata: {
nameSingular: payload.objectMetadata.nameSingular,
isCustom: payload.objectMetadata.isCustom,
},
},
);
}
}
// TODO: use objectMetadata instead of hardcoded standard objects name
const hasPositionField = (
createdObjectMetadata: CreatedObjectMetadata,
createdObjectMetadata: ObjectMetadataInterface,
): boolean => {
return (
createdObjectMetadata.isCustom ||

View File

@ -1,4 +1,5 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { WorkspaceQueryBuilderModule } from 'src/engine/api/graphql/workspace-query-builder/workspace-query-builder.module';
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
@ -6,20 +7,26 @@ import { WorkspacePreQueryHookModule } from 'src/engine/api/graphql/workspace-qu
import { workspaceQueryRunnerFactories } from 'src/engine/api/graphql/workspace-query-runner/factories';
import { RecordPositionListener } from 'src/engine/api/graphql/workspace-query-runner/listeners/record-position.listener';
import { AuthModule } from 'src/engine/modules/auth/auth.module';
import { FeatureFlagEntity } from 'src/engine/modules/feature-flag/feature-flag.entity';
import { Workspace } from 'src/engine/modules/workspace/workspace.entity';
import { WorkspaceQueryRunnerService } from './workspace-query-runner.service';
import { EntityEventsToDbListener } from './listeners/entity-events-to-db.listener';
@Module({
imports: [
AuthModule,
WorkspaceQueryBuilderModule,
WorkspaceDataSourceModule,
WorkspacePreQueryHookModule,
TypeOrmModule.forFeature([Workspace, FeatureFlagEntity], 'core'),
],
providers: [
WorkspaceQueryRunnerService,
...workspaceQueryRunnerFactories,
RecordPositionListener,
EntityEventsToDbListener,
],
exports: [WorkspaceQueryRunnerService],
})

View File

@ -246,10 +246,10 @@ export class WorkspaceQueryRunnerService {
parsedResults.forEach((record) => {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.created`, {
workspaceId,
createdRecord: this.removeNestedProperties(record),
createdObjectMetadata: {
nameSingular: objectMetadataItem.nameSingular,
isCustom: objectMetadataItem.isCustom,
recordId: record.id,
objectMetadata: objectMetadataItem,
details: {
after: record,
},
} satisfies ObjectRecordCreateEvent<any>);
});
@ -300,8 +300,12 @@ export class WorkspaceQueryRunnerService {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.updated`, {
workspaceId,
previousRecord: this.removeNestedProperties(existingRecord as Record),
updatedRecord: this.removeNestedProperties(parsedResults?.[0]),
recordId: (existingRecord as Record).id,
objectMetadata: objectMetadataItem,
details: {
before: this.removeNestedProperties(existingRecord as Record),
after: this.removeNestedProperties(parsedResults?.[0]),
},
} satisfies ObjectRecordUpdateEvent<any>);
return parsedResults?.[0];
@ -336,6 +340,12 @@ export class WorkspaceQueryRunnerService {
options,
);
// TODO: check - NO EVENT SENT?
// OK I spent 2 hours trying to implement before/after diff and
// figured out why it hasn't been implement
// Doing a findMany in that context is very hard as long as we don't
// have a proper ORM. Let's come back to this once we do (target end of April 24?)
return parsedResults;
}
@ -374,7 +384,11 @@ export class WorkspaceQueryRunnerService {
parsedResults.forEach((record) => {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.deleted`, {
workspaceId,
deletedRecord: [this.removeNestedProperties(record)],
recordId: record.id,
objectMetadata: objectMetadataItem,
details: {
before: [this.removeNestedProperties(record)],
},
} satisfies ObjectRecordDeleteEvent<any>);
});
@ -408,7 +422,11 @@ export class WorkspaceQueryRunnerService {
this.eventEmitter.emit(`${objectMetadataItem.nameSingular}.deleted`, {
workspaceId,
deletedRecord: this.removeNestedProperties(parsedResults?.[0]),
recordId: args.id,
objectMetadata: objectMetadataItem,
details: {
before: this.removeNestedProperties(parsedResults?.[0]),
},
} satisfies ObjectRecordDeleteEvent<any>);
return parsedResults?.[0];

View File

@ -8,3 +8,4 @@ export * from './string-filter.input-type';
export * from './time-filter.input-type';
export * from './uuid-filter.input-type';
export * from './boolean-filter.input-type';
export * from './json-filter.input-type';

View File

@ -0,0 +1,10 @@
import { GraphQLInputObjectType } from 'graphql';
import { FilterIs } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input/filter-is.input-type';
export const JsonFilterType = new GraphQLInputObjectType({
name: 'JsonFilter',
fields: {
is: { type: FilterIs },
},
});

View File

@ -15,6 +15,7 @@ import {
GraphQLString,
GraphQLType,
} from 'graphql';
import GraphQLJSON from 'graphql-type-json';
import {
DateScalarMode,
@ -31,6 +32,7 @@ import {
IntFilterType,
BooleanFilterType,
BigFloatFilterType,
JsonFilterType,
} from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/input';
import { OrderByDirectionType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/enum';
import { BigFloatScalarType } from 'src/engine/api/graphql/workspace-schema-builder/graphql-types/scalars';
@ -68,6 +70,7 @@ export class TypeMapperService {
[FieldMetadataType.PROBABILITY, GraphQLFloat],
[FieldMetadataType.RELATION, GraphQLID],
[FieldMetadataType.POSITION, PositionScalarType],
[FieldMetadataType.JSON, GraphQLJSON],
]);
return typeScalarMapping.get(fieldMetadataType);
@ -99,6 +102,7 @@ export class TypeMapperService {
[FieldMetadataType.PROBABILITY, FloatFilterType],
[FieldMetadataType.RELATION, UUIDFilterType],
[FieldMetadataType.POSITION, FloatFilterType],
[FieldMetadataType.JSON, JsonFilterType],
]);
return typeFilterMapping.get(fieldMetadataType);
@ -122,6 +126,7 @@ export class TypeMapperService {
[FieldMetadataType.SELECT, OrderByDirectionType],
[FieldMetadataType.MULTI_SELECT, OrderByDirectionType],
[FieldMetadataType.POSITION, OrderByDirectionType],
[FieldMetadataType.JSON, OrderByDirectionType],
]);
return typeOrderByMapping.get(fieldMetadataType);

View File

@ -1,12 +1,7 @@
import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects/base.object-metadata';
import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event';
export type CreatedObjectMetadata = {
nameSingular: string;
isCustom: boolean;
};
export class ObjectRecordCreateEvent<T extends BaseObjectMetadata> {
workspaceId: string;
createdRecord: T;
createdObjectMetadata: CreatedObjectMetadata;
export class ObjectRecordCreateEvent<T> extends ObjectRecordBaseEvent {
details: {
after: T;
};
}

View File

@ -1,6 +1,7 @@
import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects/base.object-metadata';
import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event';
export declare class ObjectRecordDeleteEvent<T extends BaseObjectMetadata> {
workspaceId: string;
deletedRecord: T;
export class ObjectRecordDeleteEvent<T> extends ObjectRecordBaseEvent {
details: {
before: T;
};
}

View File

@ -1,7 +1,8 @@
import { BaseObjectMetadata } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects/base.object-metadata';
import { ObjectRecordBaseEvent } from 'src/engine/integrations/event-emitter/types/object-record.base.event';
export class ObjectRecordUpdateEvent<T extends BaseObjectMetadata> {
workspaceId: string;
previousRecord: T;
updatedRecord: T;
export class ObjectRecordUpdateEvent<T> extends ObjectRecordBaseEvent {
details: {
before: T;
after: T;
};
}

View File

@ -0,0 +1,8 @@
import { ObjectMetadataInterface } from 'src/engine-metadata/field-metadata/interfaces/object-metadata.interface';
export class ObjectRecordBaseEvent {
workspaceId: string;
recordId: string;
objectMetadata: ObjectMetadataInterface;
details: any;
}

View File

@ -44,6 +44,7 @@ import { ObjectMetadataRepositoryModule } from 'src/engine/object-metadata-repos
import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
import { MessageParticipantObjectMetadata } from 'src/modules/messaging/standard-objects/message-participant.object-metadata';
import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-objects/message-channel.object-metadata';
import { SaveEventToDbJob } from 'src/engine/api/graphql/workspace-query-runner/jobs/save-event-to-db.job';
@Module({
imports: [
@ -130,6 +131,10 @@ import { MessageChannelObjectMetadata } from 'src/modules/messaging/standard-obj
provide: RecordPositionBackfillJob.name,
useClass: RecordPositionBackfillJob,
},
{
provide: SaveEventToDbJob.name,
useClass: SaveEventToDbJob,
},
],
})
export class JobsModule {

View File

@ -9,4 +9,5 @@ export enum MessageQueue {
calendarQueue = 'calendar-queue',
billingQueue = 'billing-queue',
recordPositionBackfillQueue = 'record-position-backfill-queue',
entityEventsToDbQueue = 'entity-events-to-db-queue',
}

View File

@ -20,14 +20,14 @@ export class AnalyticsResolver {
constructor(private readonly analyticsService: AnalyticsService) {}
@Mutation(() => Analytics)
createEvent(
@Args() createEventInput: CreateAnalyticsInput,
track(
@Args() createAnalyticsInput: CreateAnalyticsInput,
@AuthWorkspace() workspace: Workspace | undefined,
@AuthUser({ allowUndefined: true }) user: User | undefined,
@Context('req') request: Request,
) {
return this.analyticsService.create(
createEventInput,
createAnalyticsInput,
user,
workspace,
request,

View File

@ -16,6 +16,7 @@ import { Workspace } from 'src/engine/modules/workspace/workspace.entity';
export enum FeatureFlagKeys {
IsBlocklistEnabled = 'IS_BLOCKLIST_ENABLED',
IsCalendarEnabled = 'IS_CALENDAR_ENABLED',
IsEventObjectEnabled = 'IS_EVENT_OBJECT_ENABLED',
}
@Entity({ name: 'featureFlag', schema: 'core' })

View File

@ -69,6 +69,9 @@ const getSchemaComponentsProperties = (
),
};
break;
case FieldMetadataType.JSON:
type: 'object';
break;
default:
itemProperty.type = 'string';
break;

View File

@ -62,8 +62,10 @@ export class UserWorkspaceService extends TypeOrmQueryService<UserWorkspace> {
new ObjectRecordCreateEvent<WorkspaceMemberObjectMetadata>();
payload.workspaceId = workspaceId;
payload.createdRecord = new WorkspaceMemberObjectMetadata();
payload.createdRecord = workspaceMember[0];
payload.details = {
after: workspaceMember[0],
};
payload.recordId = workspaceMember[0].id;
this.eventEmitter.emit('workspaceMember.created', payload);
}

View File

@ -22,6 +22,8 @@ export const mapFieldMetadataTypeToDataType = (
return 'boolean';
case FieldMetadataType.DATE_TIME:
return 'timestamp';
case FieldMetadataType.JSON:
return 'jsonb';
case FieldMetadataType.RATING:
case FieldMetadataType.SELECT:
case FieldMetadataType.MULTI_SELECT:

View File

@ -50,6 +50,7 @@ export class AddStandardIdCommand extends CommandRunner {
{
IS_BLOCKLIST_ENABLED: true,
IS_CALENDAR_ENABLED: true,
IS_EVENT_OBJECT_ENABLED: true,
},
);
const standardFieldMetadataCollection = this.standardFieldFactory.create(
@ -61,6 +62,7 @@ export class AddStandardIdCommand extends CommandRunner {
{
IS_BLOCKLIST_ENABLED: true,
IS_CALENDAR_ENABLED: true,
IS_EVENT_OBJECT_ENABLED: true,
},
);

View File

@ -122,6 +122,7 @@ export const companyStandardFieldIds = {
opportunities: '20202020-add3-4658-8e23-d70dccb6d0ec',
favorites: '20202020-4d1d-41ac-b13b-621631298d55',
attachments: '20202020-c1b5-4120-b0f0-987ca401ed53',
events: '20202020-0414-4daf-9c0d-64fe7b27f89f',
};
export const connectedAccountStandardFieldIds = {
@ -135,6 +136,15 @@ export const connectedAccountStandardFieldIds = {
calendarChannels: '20202020-af4a-47bb-99ec-51911c1d3977',
};
export const eventStandardFieldIds = {
properties: '20202020-f142-4b04-b91b-6a2b4af3bf10',
workspaceMember: '20202020-af23-4479-9a30-868edc474b35',
person: '20202020-c414-45b9-a60a-ac27aa96229e',
company: '20202020-04ad-4221-a744-7a8278a5ce20',
opportunity: '20202020-7664-4a35-a3df-580d389fd5f0',
custom: '20202020-4a71-41b0-9f83-9cdcca3f8b14',
};
export const favoriteStandardFieldIds = {
position: '20202020-dd26-42c6-8c3c-2a7598c204f6',
workspaceMember: '20202020-ce63-49cb-9676-fdc0c45892cd',
@ -199,6 +209,7 @@ export const opportunityStandardFieldIds = {
favorites: '20202020-a1c2-4500-aaae-83ba8a0e827a',
activityTargets: '20202020-220a-42d6-8261-b2102d6eab35',
attachments: '20202020-87c7-4118-83d6-2f4031005209',
events: '20202020-30e2-421f-96c7-19c69d1cf631',
};
export const personStandardFieldIds = {
@ -218,6 +229,7 @@ export const personStandardFieldIds = {
attachments: '20202020-cd97-451f-87fa-bcb789bdbf3a',
messageParticipants: '20202020-498e-4c61-8158-fa04f0638334',
calendarEventAttendees: '20202020-52ee-45e9-a702-b64b3753e3a9',
events: '20202020-a43e-4873-9c23-e522de906ce5',
};
export const pipelineStepStandardFieldIds = {
@ -284,6 +296,7 @@ export const workspaceMemberStandardFieldIds = {
messageParticipants: '20202020-8f99-48bc-a5eb-edd33dd54188',
blocklist: '20202020-6cb2-4161-9f29-a4b7f1283859',
calendarEventAttendees: '20202020-0dbc-4841-9ce1-3e793b5b3512',
events: '20202020-e15b-47b8-94fe-8200e3c66615',
};
export const customObjectStandardFieldIds = {

View File

@ -18,6 +18,7 @@ export const standardObjectIds = {
comment: '20202020-435f-4de9-89b5-97e32233bf5f',
company: '20202020-b374-4779-a561-80086cb2e17f',
connectedAccount: '20202020-977e-46b2-890b-c3002ddfd5c5',
event: '20202020-6736-4337-b5c4-8b39fae325a5',
favorite: '20202020-ab56-4e05-92a3-e2414a499860',
messageChannelMessageAssociation: '20202020-ad1e-4127-bccb-d83ae04d2ccb',
messageChannel: '20202020-fe8c-40bc-a681-b80b771449b7',

View File

@ -25,6 +25,7 @@ import { ViewObjectMetadata } from 'src/modules/view/standard-objects/view.objec
import { WebhookObjectMetadata } from 'src/modules/webhook/standard-objects/webhook.object-metadata';
import { WorkspaceMemberObjectMetadata } from 'src/modules/workspace-member/standard-objects/workspace-member.object-metadata';
import { CalendarChannelEventAssociationObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel-event-association.object-metadata';
import { EventObjectMetadata } from 'src/modules/event/standard-objects/event.object-metadata';
export const standardObjectMetadataDefinitions = [
ActivityTargetObjectMetadata,
@ -35,6 +36,7 @@ export const standardObjectMetadataDefinitions = [
CommentObjectMetadata,
CompanyObjectMetadata,
ConnectedAccountObjectMetadata,
EventObjectMetadata,
FavoriteObjectMetadata,
OpportunityObjectMetadata,
PersonObjectMetadata,