admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Update clean inactive workspaces (#3600)

Browse Source 
* Fix typo

* Add dry-run option in clean inactive workspaces

* Add logs

* Chunk workspace metadata

* Add BCC to clean workspace notification email

* Send workspace to delete ids in one email

* Update example

* Update function naming
This commit is contained in:
martmull
2024-01-24 12:51:42 +01:00
committed by GitHub
parent f48814f6d9
commit b991790f62
8 changed files with 157 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/twenty-docs/docs/start/self-hosting/environment-variables.mdx
View File

@ -62,7 +62,7 @@ import TabItem from '@theme/TabItem';
### Email ### Email
<OptionTable options={[ <OptionTable options={[
['EMAIL_FROM_ADDRESS', 'noreply@yourdomain.com', 'Global email From: header used to send emails'], ['EMAIL_FROM_ADDRESS', 'contact@yourdomain.com', 'Global email From: header used to send emails'],
['EMAIL_FROM_NAME', 'John from YourDomain', 'Global name From: header used to send emails'], ['EMAIL_FROM_NAME', 'John from YourDomain', 'Global name From: header used to send emails'],
['EMAIL_SYSTEM_ADDRESS', 'system@yourdomain.com', 'Email address used as a destination to send internal system notification'], ['EMAIL_SYSTEM_ADDRESS', 'system@yourdomain.com', 'Email address used as a destination to send internal system notification'],
['EMAIL_DRIVER', 'logger', "Email driver: 'logger' (to log emails in console) or 'smtp'"], ['EMAIL_DRIVER', 'logger', "Email driver: 'logger' (to log emails in console) or 'smtp'"],

4
packages/twenty-emails/src/components/BaseEmail.tsx
View File

@ -3,11 +3,11 @@ import { Container, Html } from '@react-email/components';
import { BaseHead } from 'src/components/BaseHead'; import { BaseHead } from 'src/components/BaseHead';
import { Logo } from 'src/components/Logo'; import { Logo } from 'src/components/Logo';
export const BaseEmail = ({ children }) => { export const BaseEmail = ({ children, width = 290 }) => {
return ( return (
<Html lang="en"> <Html lang="en">
<BaseHead /> <BaseHead />
<Container width={290}> <Container width={width}>
<Logo /> <Logo />
{children} {children}
</Container> </Container>

35
packages/twenty-emails/src/emails/delete-inactive-workspaces.email.tsx
View File

@ -1,27 +1,38 @@
import * as React from 'react'; import * as React from 'react';
import { Column, Row, Section } from '@react-email/components';
import { BaseEmail } from 'src/components/BaseEmail'; import { BaseEmail } from 'src/components/BaseEmail';
import { CallToAction } from 'src/components/CallToAction';
import { HighlightedText } from 'src/components/HighlightedText';
import { MainText } from 'src/components/MainText'; import { MainText } from 'src/components/MainText';
import { Title } from 'src/components/Title'; import { Title } from 'src/components/Title';
type DeleteInactiveWorkspaceEmailData = { type DeleteInactiveWorkspaceEmailData = {
daysSinceDead: number; daysSinceInactive: number;
workspaceId: string; workspaceId: string;
}; };
export const DeleteInactiveWorkspaceEmail = ({ export const DeleteInactiveWorkspaceEmail = (
daysSinceDead, workspacesToDelete: DeleteInactiveWorkspaceEmailData[],
workspaceId, ) => {
}: DeleteInactiveWorkspaceEmailData) => { const minDaysSinceInactive = Math.min(
...workspacesToDelete.map(
(workspaceToDelete) => workspaceToDelete.daysSinceInactive,
),
);
return ( return (
<BaseEmail> <BaseEmail width={350}>
<Title value="Dead Workspace 😵" /> <Title value="Dead Workspaces 😵 that should be deleted" />
<HighlightedText value={`Inactive since ${daysSinceDead} day(s)`} />
<MainText> <MainText>
Workspace <b>{workspaceId}</b> should be deleted. List of <b>workspaceIds</b> inactive since at least{' '}
<b>{minDaysSinceInactive} days</b>:
<Section>
{workspacesToDelete.map((workspaceToDelete) => {
return (
<Row key={workspaceToDelete.workspaceId}>
<Column>{workspaceToDelete.workspaceId}</Column>
</Row>
);
})}
</Section>
</MainText> </MainText>
<CallToAction href="https://app.twenty.com" value="Connect to Twenty" />
</BaseEmail> </BaseEmail>
); );
}; };

2
packages/twenty-server/.env.example
View File

@ -43,7 +43,7 @@ SIGN_IN_PREFILLED=true
# WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION=30 # WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION=30
# WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION=60 # WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION=60
# Email Server Settings, see this doc for more info: https://docs.twenty.com/start/self-hosting/environment-variables # Email Server Settings, see this doc for more info: https://docs.twenty.com/start/self-hosting/environment-variables
# EMAIL_FROM_ADDRESS=noreply@yourdomain.com # EMAIL_FROM_ADDRESS=contact@yourdomain.com
# EMAIL_SYSTEM_ADDRESS=system@yourdomain.com # EMAIL_SYSTEM_ADDRESS=system@yourdomain.com
# EMAIL_FROM_NAME='John from YourDomain' # EMAIL_FROM_NAME='John from YourDomain'
# EMAIL_DRIVER=logger # EMAIL_DRIVER=logger

140
packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job.ts
View File

@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm'; import { InjectRepository } from '@nestjs/typeorm';
import { render } from '@react-email/render'; import { render } from '@react-email/render';
import { Repository } from 'typeorm'; import { In, Repository } from 'typeorm';
import { import {
CleanInactiveWorkspaceEmail, CleanInactiveWorkspaceEmail,
DeleteInactiveWorkspaceEmail, DeleteInactiveWorkspaceEmail,
@ -23,14 +23,23 @@ import {
} from 'src/core/feature-flag/feature-flag.entity'; } from 'src/core/feature-flag/feature-flag.entity';
import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity'; import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util'; import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
import { CleanInactiveWorkspacesCommandOptions } from 'src/workspace/cron/clean-inactive-workspaces/commands/clean-inactive-workspaces.command';
const MILLISECONDS_IN_ONE_DAY = 1000 * 3600 * 24; const MILLISECONDS_IN_ONE_DAY = 1000 * 3600 * 24;
type WorkspaceToDeleteData = {
workspaceId: string;
daysSinceInactive: number;
};
@Injectable() @Injectable()
export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> { export class CleanInactiveWorkspaceJob
implements MessageQueueJob<CleanInactiveWorkspacesCommandOptions>
{
private readonly logger = new Logger(CleanInactiveWorkspaceJob.name); private readonly logger = new Logger(CleanInactiveWorkspaceJob.name);
private readonly inactiveDaysBeforeDelete; private readonly inactiveDaysBeforeDelete;
private readonly inactiveDaysBeforeEmail; private readonly inactiveDaysBeforeEmail;
private workspacesToDelete: WorkspaceToDeleteData[] = [];
constructor( constructor(
private readonly dataSourceService: DataSourceService, private readonly dataSourceService: DataSourceService,
@ -48,7 +57,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
this.environmentService.getInactiveDaysBeforeEmail(); this.environmentService.getInactiveDaysBeforeEmail();
} }
async getmostRecentUpdatedAt( async getMostRecentUpdatedAt(
dataSource: DataSourceEntity, dataSource: DataSourceEntity,
objectsMetadata: ObjectMetadataEntity[], objectsMetadata: ObjectMetadataEntity[],
): Promise<Date> { ): Promise<Date> {
@ -89,6 +98,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
async warnWorkspaceUsers( async warnWorkspaceUsers(
dataSource: DataSourceEntity, dataSource: DataSourceEntity,
daysSinceInactive: number, daysSinceInactive: number,
isDryRun: boolean,
) { ) {
const workspaceMembers = const workspaceMembers =
await this.userService.loadWorkspaceMembers(dataSource); await this.userService.loadWorkspaceMembers(dataSource);
@ -103,13 +113,17 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
)?.[0].displayName; )?.[0].displayName;
this.logger.log( this.logger.log(
`Sending workspace ${ `${this.getDryRunLogHeader(isDryRun)}Sending workspace ${
dataSource.workspaceId dataSource.workspaceId
} inactive since ${daysSinceInactive} days emails to users ['${workspaceMembers } inactive since ${daysSinceInactive} days emails to users ['${workspaceMembers
.map((workspaceUser) => workspaceUser.email) .map((workspaceUser) => workspaceUser.email)
.join(', ')}']`, .join(', ')}']`,
); );
if (isDryRun) {
return;
}
workspaceMembers.forEach((workspaceMember) => { workspaceMembers.forEach((workspaceMember) => {
const emailData = { const emailData = {
daysLeft: this.inactiveDaysBeforeDelete - daysSinceInactive, daysLeft: this.inactiveDaysBeforeDelete - daysSinceInactive,
@ -126,6 +140,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
this.emailService.send({ this.emailService.send({
to: workspaceMember.email, to: workspaceMember.email,
bcc: this.environmentService.getEmailSystemAddress(),
from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`, from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
subject: 'Action Needed to Prevent Workspace Deletion', subject: 'Action Needed to Prevent Workspace Deletion',
html, html,
@ -137,36 +152,32 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
async deleteWorkspace( async deleteWorkspace(
dataSource: DataSourceEntity, dataSource: DataSourceEntity,
daysSinceInactive: number, daysSinceInactive: number,
isDryRun: boolean,
): Promise<void> { ): Promise<void> {
this.logger.log( this.logger.log(
`Sending email to delete workspace ${dataSource.workspaceId} inactive since ${daysSinceInactive} days`, `${this.getDryRunLogHeader(isDryRun)}Sending email to delete workspace ${
dataSource.workspaceId
} inactive since ${daysSinceInactive} days`,
); );
if (isDryRun) {
return;
}
const emailData = { const emailData = {
daysSinceDead: daysSinceInactive - this.inactiveDaysBeforeDelete, daysSinceInactive: daysSinceInactive,
workspaceId: `${dataSource.workspaceId}`, workspaceId: `${dataSource.workspaceId}`,
}; };
const emailTemplate = DeleteInactiveWorkspaceEmail(emailData);
const html = render(emailTemplate, {
pretty: true,
});
const text = `Workspace '${dataSource.workspaceId}' should be deleted as inactive since ${daysSinceInactive} days`; this.workspacesToDelete.push(emailData);
await this.emailService.send({
to: this.environmentService.getEmailSystemAddress(),
from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
subject: 'Action Needed to Delete Workspace',
html,
text,
});
} }
async processWorkspace( async processWorkspace(
dataSource: DataSourceEntity, dataSource: DataSourceEntity,
objectsMetadata: ObjectMetadataEntity[], objectsMetadata: ObjectMetadataEntity[],
isDryRun: boolean,
): Promise<void> { ): Promise<void> {
const mostRecentUpdatedAt = await this.getmostRecentUpdatedAt( const mostRecentUpdatedAt = await this.getMostRecentUpdatedAt(
dataSource, dataSource,
objectsMetadata, objectsMetadata,
); );
@ -176,9 +187,9 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
); );
if (daysSinceInactive > this.inactiveDaysBeforeDelete) { if (daysSinceInactive > this.inactiveDaysBeforeDelete) {
await this.deleteWorkspace(dataSource, daysSinceInactive); await this.deleteWorkspace(dataSource, daysSinceInactive, isDryRun);
} else if (daysSinceInactive > this.inactiveDaysBeforeEmail) { } else if (daysSinceInactive > this.inactiveDaysBeforeEmail) {
await this.warnWorkspaceUsers(dataSource, daysSinceInactive); await this.warnWorkspaceUsers(dataSource, daysSinceInactive, isDryRun);
} }
} }
@ -196,8 +207,47 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
); );
} }
async handle(): Promise<void> { getDryRunLogHeader(isDryRun: boolean): string {
this.logger.log('Job running...'); return isDryRun ? 'Dry-run mode: ' : '';
}
chunkArray(array: any[], chunkSize = 6): any[][] {
const chunkedArray: any[][] = [];
let index = 0;
while (index < array.length) {
chunkedArray.push(array.slice(index, index + chunkSize));
index += chunkSize;
}
return chunkedArray;
}
async sendDeleteWorkspaceEmail(isDryRun: boolean): Promise<void> {
if (isDryRun || this.workspacesToDelete.length === 0) {
return;
}
const emailTemplate = DeleteInactiveWorkspaceEmail(this.workspacesToDelete);
const html = render(emailTemplate, {
pretty: true,
});
const text = render(emailTemplate, {
plainText: true,
});
await this.emailService.send({
to: this.environmentService.getEmailSystemAddress(),
from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
subject: 'Action Needed to Delete Workspaces',
html,
text,
});
}
async handle(data: CleanInactiveWorkspacesCommandOptions): Promise<void> {
const isDryRun = data.dryRun || false;
this.logger.log(`${this.getDryRunLogHeader(isDryRun)}Job running...`);
if (!this.inactiveDaysBeforeDelete && !this.inactiveDaysBeforeEmail) { if (!this.inactiveDaysBeforeDelete && !this.inactiveDaysBeforeEmail) {
this.logger.log( this.logger.log(
`'WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION' and 'WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION' environment variables not set, please check this doc for more info: https://docs.twenty.com/start/self-hosting/environment-variables`, `'WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION' and 'WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION' environment variables not set, please check this doc for more info: https://docs.twenty.com/start/self-hosting/environment-variables`,
@ -205,20 +255,46 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
return; return;
} }
const dataSources = const dataSources =
await this.dataSourceService.getManyDataSourceMetadata(); await this.dataSourceService.getManyDataSourceMetadata();
const objectsMetadata = await this.objectMetadataService.findMany(); const dataSourcesChunks = this.chunkArray(dataSources);
for (const dataSource of dataSources) { this.logger.log(
if (!(await this.isWorkspaceCleanable(dataSource))) { `${this.getDryRunLogHeader(isDryRun)}On ${
continue; dataSources.length
} workspaces divided in ${dataSourcesChunks.length} chunks...`,
);
for (const dataSourcesChunk of dataSourcesChunks) {
const objectsMetadata = await this.objectMetadataService.findMany({
where: {
dataSourceId: In(dataSourcesChunk.map((dataSource) => dataSource.id)),
},
});
for (const dataSource of dataSourcesChunk) {
if (!(await this.isWorkspaceCleanable(dataSource))) {
this.logger.log(
`${this.getDryRunLogHeader(isDryRun)}Workspace ${
dataSource.workspaceId
} not cleanable`,
);
continue;
}
this.logger.log(
`${this.getDryRunLogHeader(isDryRun)}Cleaning Workspace ${
dataSource.workspaceId
}`,
);
await this.processWorkspace(dataSource, objectsMetadata, isDryRun);
} }
this.logger.log(`Cleaning Workspace ${dataSource.workspaceId}`);
await this.processWorkspace(dataSource, objectsMetadata);
} }
this.logger.log('job done!'); await this.sendDeleteWorkspaceEmail(isDryRun);
this.logger.log(`${this.getDryRunLogHeader(isDryRun)}job done!`);
} }
} }

24
packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/clean-inactive-workspaces.command.ts
View File

@ -1,11 +1,15 @@
import { Inject } from '@nestjs/common'; import { Inject } from '@nestjs/common';
import { Command, CommandRunner } from 'nest-commander'; import { Command, CommandRunner, Option } from 'nest-commander';
import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants'; import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service'; import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job'; import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';
export type CleanInactiveWorkspacesCommandOptions = {
dryRun: boolean;
};
@Command({ @Command({
name: 'clean-inactive-workspaces', name: 'clean-inactive-workspaces',
description: 'Clean inactive workspaces', description: 'Clean inactive workspaces',
@ -18,10 +22,22 @@ export class CleanInactiveWorkspacesCommand extends CommandRunner {
super(); super();
} }
async run(): Promise<void> { @Option({
await this.messageQueueService.add<any>( flags: '-d, --dry-run [dry run]',
description: 'Dry run: Log cleaning actions without executing them.',
required: false,
})
dryRun(value: string): boolean {
return Boolean(value);
}
async run(
_passedParam: string[],
options: CleanInactiveWorkspacesCommandOptions,
): Promise<void> {
await this.messageQueueService.add<CleanInactiveWorkspacesCommandOptions>(
CleanInactiveWorkspaceJob.name, CleanInactiveWorkspaceJob.name,
{}, options,
{ retryLimit: 3 }, { retryLimit: 3 },
); );
} }

2
packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/start-clean-inactive-workspaces.cron.command.ts
View File

@ -8,7 +8,7 @@ import { cleanInactiveWorkspaceCronPattern } from 'src/workspace/cron/clean-inac
import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job'; import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';
@Command({ @Command({
name: 'clean-inactive-workspace:cron:start', name: 'clean-inactive-workspaces:cron:start',
description: 'Starts a cron job to clean inactive workspaces', description: 'Starts a cron job to clean inactive workspaces',
}) })
export class StartCleanInactiveWorkspacesCronCommand extends CommandRunner { export class StartCleanInactiveWorkspacesCronCommand extends CommandRunner {

2
packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/stop-clean-inactive-workspaces.cron.command.ts
View File

@ -8,7 +8,7 @@ import { cleanInactiveWorkspaceCronPattern } from 'src/workspace/cron/clean-inac
import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job'; import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';
@Command({ @Command({
name: 'clean-inactive-workspace:cron:stop', name: 'clean-inactive-workspaces:cron:stop',
description: 'Stops the clean inactive workspaces cron job', description: 'Stops the clean inactive workspaces cron job',
}) })
export class StopCleanInactiveWorkspacesCronCommand extends CommandRunner { export class StopCleanInactiveWorkspacesCronCommand extends CommandRunner {