3011 fill the messagerecipient table when fetching messages (#3073)
* wip * trying to parse display names and emails * add nodemailer mailparser * mail parsing is working * add personId and workspaceMemberId * add date to messages * Fix PR * Run tsc on bigger machine * Fix lint --------- Co-authored-by: Charles Bochet <charles@twenty.com>
This commit is contained in:
bosiraphael
@ -9,6 +9,7 @@
|
||||
"build": "tsc && vite build && yarn build:inject-runtime-env",
|
||||
"build:inject-runtime-env": "sh ./scripts/inject-runtime-env.sh",
|
||||
"tsc": "tsc --watch",
|
||||
"tsc:ci": "tsc",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
|
||||
"lint:ci": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0 --config .eslintrc-ci.cjs",
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { H2Title } from '@/ui/display/typography/components/H2Title';
|
||||
import { Section } from '@/ui/layout/section/components/Section';
|
||||
|
||||
import { SettingsAccountsEmptyStateCard } from './SettingsAccountsEmptyStateCard';
|
||||
|
||||
export const SettingsNewAccountSection = () => {
|
||||
|
||||
@ -3,4 +3,5 @@ export type FeatureFlagKey =
|
||||
| 'IS_NOTE_CREATE_IMAGES_ENABLED'
|
||||
| 'IS_RELATION_FIELD_TYPE_ENABLED'
|
||||
| 'IS_SELECT_FIELD_TYPE_ENABLED'
|
||||
| 'IS_QUICK_ACTIONS_ENABLED'
|
||||
| 'IS_RATING_FIELD_TYPE_ENABLED';
|
||||
|
||||
@ -58,6 +58,7 @@
|
||||
"@sentry/tracing": "^7.66.0",
|
||||
"@types/lodash.camelcase": "^4.3.7",
|
||||
"@types/lodash.merge": "^4.6.7",
|
||||
"@types/mailparser": "^3.4.4",
|
||||
"add": "^2.0.6",
|
||||
"apollo-server-express": "^3.12.0",
|
||||
"axios": "^1.6.2",
|
||||
@ -88,6 +89,7 @@
|
||||
"lodash.merge": "^4.6.2",
|
||||
"lodash.snakecase": "^4.1.1",
|
||||
"lodash.upperfirst": "^4.3.1",
|
||||
"mailparser": "^3.6.5",
|
||||
"microdiff": "^1.3.2",
|
||||
"nest-commander": "^3.12.0",
|
||||
"openapi-types": "^12.1.3",
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import axios, { AxiosInstance } from 'axios';
|
||||
import { simpleParser } from 'mailparser';
|
||||
|
||||
@Injectable()
|
||||
export class FetchBatchMessagesService {
|
||||
@ -57,7 +58,9 @@ export class FetchBatchMessagesService {
|
||||
},
|
||||
);
|
||||
|
||||
return this.formatBatchResponse(response);
|
||||
const formattedResponse = await this.formatBatchResponse(response);
|
||||
|
||||
return formattedResponse;
|
||||
}
|
||||
|
||||
createBatchBody(messageQueries, boundary: string): string {
|
||||
@ -121,40 +124,44 @@ export class FetchBatchMessagesService {
|
||||
return boundary.replace('boundary=', '').trim('; ');
|
||||
}
|
||||
|
||||
formatBatchResponse(response) {
|
||||
async formatBatchResponse(response) {
|
||||
const parsedResponse = this.parseBatch(response);
|
||||
|
||||
return parsedResponse
|
||||
.map((item) => {
|
||||
const { id, threadId, payload } = item;
|
||||
return Promise.all(
|
||||
parsedResponse.map(async (item) => {
|
||||
const { id, threadId, internalDate, raw } = item;
|
||||
|
||||
const headers = payload?.headers;
|
||||
const message = atob(raw?.replace(/-/g, '+').replace(/_/g, '/'));
|
||||
|
||||
const parts = payload?.parts;
|
||||
const parsed = await simpleParser(message);
|
||||
|
||||
if (!parts) {
|
||||
return;
|
||||
}
|
||||
|
||||
const bodyBase64 = parts[0]?.body?.data;
|
||||
|
||||
if (!bodyBase64) {
|
||||
return;
|
||||
}
|
||||
|
||||
const body = atob(bodyBase64.replace(/-/g, '+').replace(/_/g, '/'));
|
||||
const {
|
||||
subject,
|
||||
messageId,
|
||||
from,
|
||||
to,
|
||||
cc,
|
||||
bcc,
|
||||
text,
|
||||
html,
|
||||
attachments,
|
||||
} = parsed;
|
||||
|
||||
return {
|
||||
externalId: id,
|
||||
headerMessageId: headers?.find(
|
||||
(header) => header.name === 'Message-ID',
|
||||
)?.value,
|
||||
subject: headers?.find((header) => header.name === 'Subject')?.value,
|
||||
headerMessageId: messageId,
|
||||
subject: subject,
|
||||
messageThreadId: threadId,
|
||||
from: headers?.find((header) => header.name === 'From')?.value,
|
||||
body,
|
||||
internalDate,
|
||||
from,
|
||||
to,
|
||||
cc,
|
||||
bcc,
|
||||
text,
|
||||
html,
|
||||
attachments,
|
||||
};
|
||||
})
|
||||
.filter((item) => item);
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { google } from 'googleapis';
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { TypeORMService } from 'src/database/typeorm/typeorm.service';
|
||||
import { EnvironmentService } from 'src/integrations/environment/environment.service';
|
||||
@ -110,7 +111,7 @@ export class FetchWorkspaceMessagesService {
|
||||
}
|
||||
|
||||
const messageQueries = messagesData.map((message) => ({
|
||||
uri: '/gmail/v1/users/me/messages/' + message.id,
|
||||
uri: '/gmail/v1/users/me/messages/' + message.id + '?format=RAW',
|
||||
}));
|
||||
|
||||
const messagesResponse =
|
||||
@ -123,6 +124,7 @@ export class FetchWorkspaceMessagesService {
|
||||
messagesResponse,
|
||||
dataSourceMetadata,
|
||||
workspaceDataSource,
|
||||
workspaceMemberId,
|
||||
);
|
||||
|
||||
return messages;
|
||||
@ -168,33 +170,61 @@ export class FetchWorkspaceMessagesService {
|
||||
}
|
||||
}
|
||||
|
||||
async saveMessages(messages, dataSourceMetadata, workspaceDataSource) {
|
||||
async saveMessages(
|
||||
messages,
|
||||
dataSourceMetadata,
|
||||
workspaceDataSource,
|
||||
workspaceMemberId,
|
||||
) {
|
||||
for (const message of messages) {
|
||||
const {
|
||||
externalId,
|
||||
headerMessageId,
|
||||
subject,
|
||||
messageThreadId,
|
||||
internalDate,
|
||||
from,
|
||||
body,
|
||||
text,
|
||||
} = message;
|
||||
|
||||
const date = new Date(parseInt(internalDate));
|
||||
|
||||
const messageThread = await workspaceDataSource?.query(
|
||||
`SELECT * FROM ${dataSourceMetadata.schema}."messageThread" WHERE "externalId" = $1`,
|
||||
[messageThreadId],
|
||||
);
|
||||
|
||||
await workspaceDataSource?.query(
|
||||
`INSERT INTO ${dataSourceMetadata.schema}."message" ("externalId", "headerMessageId", "subject", "messageThreadId", "direction", "body") VALUES ($1, $2, $3, $4, $5, $6)`,
|
||||
[
|
||||
externalId,
|
||||
headerMessageId,
|
||||
subject,
|
||||
messageThread[0]?.id,
|
||||
'incoming',
|
||||
body,
|
||||
],
|
||||
const messageId = v4();
|
||||
const handle = from?.value[0]?.address;
|
||||
const displayName = from?.value[0]?.name;
|
||||
|
||||
const person = await workspaceDataSource?.query(
|
||||
`SELECT * FROM ${dataSourceMetadata.schema}."person" WHERE "email" = $1`,
|
||||
[handle],
|
||||
);
|
||||
|
||||
const personId = person[0]?.id;
|
||||
|
||||
await workspaceDataSource?.transaction(async (manager) => {
|
||||
await manager.query(
|
||||
`INSERT INTO ${dataSourceMetadata.schema}."message" ("id", "externalId", "headerMessageId", "subject", "date", "messageThreadId", "direction", "body") VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
||||
[
|
||||
messageId,
|
||||
externalId,
|
||||
headerMessageId,
|
||||
subject,
|
||||
date,
|
||||
messageThread[0]?.id,
|
||||
'incoming',
|
||||
text,
|
||||
],
|
||||
);
|
||||
|
||||
await manager.query(
|
||||
`INSERT INTO ${dataSourceMetadata.schema}."messageRecipient" ("messageId", "role", "handle", "displayName", "personId", "workspaceMemberId") VALUES ($1, $2, $3, $4, $5, $6)`,
|
||||
[messageId, 'from', handle, displayName, personId, workspaceMemberId],
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -81,6 +81,15 @@ export class MessageObjectMetadata extends BaseObjectMetadata {
|
||||
@IsNullable()
|
||||
body: string;
|
||||
|
||||
@FieldMetadata({
|
||||
type: FieldMetadataType.DATE_TIME,
|
||||
label: 'Date',
|
||||
description: 'Date',
|
||||
icon: 'IconCalendar',
|
||||
})
|
||||
@IsNullable()
|
||||
date: string;
|
||||
|
||||
@FieldMetadata({
|
||||
type: FieldMetadataType.RELATION,
|
||||
label: 'Message Recipients',
|
||||
|
||||
Reference in New Issue
Block a user