Update clean inactive workspaces (#3600)

* Fix typo * Add dry-run option in clean inactive workspaces * Add logs * Chunk workspace metadata * Add BCC to clean workspace notification email * Send workspace to delete ids in one email * Update example * Update function naming
2024-01-24 12:51:42 +01:00
parent f48814f6d9
commit b991790f62
8 changed files with 157 additions and 54 deletions
--- a/packages/twenty-docs/docs/start/self-hosting/environment-variables.mdx
+++ b/packages/twenty-docs/docs/start/self-hosting/environment-variables.mdx
@ -62,7 +62,7 @@ import TabItem from '@theme/TabItem';
 ### Email

 <OptionTable options={[
-    ['EMAIL_FROM_ADDRESS', 'noreply@yourdomain.com', 'Global email From: header used to send emails'],
+    ['EMAIL_FROM_ADDRESS', 'contact@yourdomain.com', 'Global email From: header used to send emails'],
    ['EMAIL_FROM_NAME', 'John from YourDomain', 'Global name From: header used to send emails'],
    ['EMAIL_SYSTEM_ADDRESS', 'system@yourdomain.com', 'Email address used as a destination to send internal system notification'],
    ['EMAIL_DRIVER', 'logger', "Email driver: 'logger' (to log emails in console) or 'smtp'"],
--- a/packages/twenty-emails/src/components/BaseEmail.tsx
+++ b/packages/twenty-emails/src/components/BaseEmail.tsx
@ -3,11 +3,11 @@ import { Container, Html } from '@react-email/components';
 import { BaseHead } from 'src/components/BaseHead';
 import { Logo } from 'src/components/Logo';

-export const BaseEmail = ({ children }) => {
+export const BaseEmail = ({ children, width = 290 }) => {
  return (
    <Html lang="en">
      <BaseHead />
-      <Container width={290}>
+      <Container width={width}>
        <Logo />
        {children}
      </Container>
--- a/packages/twenty-emails/src/emails/delete-inactive-workspaces.email.tsx
+++ b/packages/twenty-emails/src/emails/delete-inactive-workspaces.email.tsx
@ -1,27 +1,38 @@
 import * as React from 'react';
+import { Column, Row, Section } from '@react-email/components';
 import { BaseEmail } from 'src/components/BaseEmail';
-import { CallToAction } from 'src/components/CallToAction';
-import { HighlightedText } from 'src/components/HighlightedText';
 import { MainText } from 'src/components/MainText';
 import { Title } from 'src/components/Title';

 type DeleteInactiveWorkspaceEmailData = {
-  daysSinceDead: number;
+  daysSinceInactive: number;
  workspaceId: string;
 };

-export const DeleteInactiveWorkspaceEmail = ({
-  daysSinceDead,
-  workspaceId,
-}: DeleteInactiveWorkspaceEmailData) => {
+export const DeleteInactiveWorkspaceEmail = (
+  workspacesToDelete: DeleteInactiveWorkspaceEmailData[],
+) => {
+  const minDaysSinceInactive = Math.min(
+    ...workspacesToDelete.map(
+      (workspaceToDelete) => workspaceToDelete.daysSinceInactive,
+    ),
+  );
  return (
-    <BaseEmail>
-      <Title value="Dead Workspace 😵" />
-      <HighlightedText value={`Inactive since ${daysSinceDead} day(s)`} />
+    <BaseEmail width={350}>
+      <Title value="Dead Workspaces 😵 that should be deleted" />
      <MainText>
-        Workspace <b>{workspaceId}</b> should be deleted.
+        List of <b>workspaceIds</b> inactive since at least{' '}
+        <b>{minDaysSinceInactive} days</b>:
+        <Section>
+          {workspacesToDelete.map((workspaceToDelete) => {
+            return (
+              <Row key={workspaceToDelete.workspaceId}>
+                <Column>{workspaceToDelete.workspaceId}</Column>
+              </Row>
+            );
+          })}
+        </Section>
      </MainText>
-      <CallToAction href="https://app.twenty.com" value="Connect to Twenty" />
    </BaseEmail>
  );
 };
--- a/packages/twenty-server/.env.example
+++ b/packages/twenty-server/.env.example
@ -43,7 +43,7 @@ SIGN_IN_PREFILLED=true
 # WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION=30
 # WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION=60
 # Email Server Settings, see this doc for more info: https://docs.twenty.com/start/self-hosting/environment-variables
-# EMAIL_FROM_ADDRESS=noreply@yourdomain.com
+# EMAIL_FROM_ADDRESS=contact@yourdomain.com
 # EMAIL_SYSTEM_ADDRESS=system@yourdomain.com
 # EMAIL_FROM_NAME='John from YourDomain'
 # EMAIL_DRIVER=logger
--- a/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job.ts
+++ b/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job.ts
@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';

 import { render } from '@react-email/render';
-import { Repository } from 'typeorm';
+import { In, Repository } from 'typeorm';
 import {
  CleanInactiveWorkspaceEmail,
  DeleteInactiveWorkspaceEmail,
@ -23,14 +23,23 @@ import {
 } from 'src/core/feature-flag/feature-flag.entity';
 import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
 import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
+import { CleanInactiveWorkspacesCommandOptions } from 'src/workspace/cron/clean-inactive-workspaces/commands/clean-inactive-workspaces.command';

 const MILLISECONDS_IN_ONE_DAY = 1000 * 3600 * 24;

+type WorkspaceToDeleteData = {
+  workspaceId: string;
+  daysSinceInactive: number;
+};
+
@Injectable()
-export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
+export class CleanInactiveWorkspaceJob
+  implements MessageQueueJob<CleanInactiveWorkspacesCommandOptions>
+{
  private readonly logger = new Logger(CleanInactiveWorkspaceJob.name);
  private readonly inactiveDaysBeforeDelete;
  private readonly inactiveDaysBeforeEmail;
+  private workspacesToDelete: WorkspaceToDeleteData[] = [];

  constructor(
    private readonly dataSourceService: DataSourceService,
@ -48,7 +57,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
      this.environmentService.getInactiveDaysBeforeEmail();
  }

-  async getmostRecentUpdatedAt(
+  async getMostRecentUpdatedAt(
    dataSource: DataSourceEntity,
    objectsMetadata: ObjectMetadataEntity[],
  ): Promise<Date> {
@ -89,6 +98,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
  async warnWorkspaceUsers(
    dataSource: DataSourceEntity,
    daysSinceInactive: number,
+    isDryRun: boolean,
  ) {
    const workspaceMembers =
      await this.userService.loadWorkspaceMembers(dataSource);
@ -103,13 +113,17 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
    )?.[0].displayName;

    this.logger.log(
-      `Sending workspace ${
+      `${this.getDryRunLogHeader(isDryRun)}Sending workspace ${
        dataSource.workspaceId
      } inactive since ${daysSinceInactive} days emails to users ['${workspaceMembers
        .map((workspaceUser) => workspaceUser.email)
        .join(', ')}']`,
    );

+    if (isDryRun) {
+      return;
+    }
+
    workspaceMembers.forEach((workspaceMember) => {
      const emailData = {
        daysLeft: this.inactiveDaysBeforeDelete - daysSinceInactive,
@ -126,6 +140,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {

      this.emailService.send({
        to: workspaceMember.email,
+        bcc: this.environmentService.getEmailSystemAddress(),
        from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
        subject: 'Action Needed to Prevent Workspace Deletion',
        html,
@ -137,36 +152,32 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
  async deleteWorkspace(
    dataSource: DataSourceEntity,
    daysSinceInactive: number,
+    isDryRun: boolean,
  ): Promise<void> {
    this.logger.log(
-      `Sending email to delete workspace ${dataSource.workspaceId} inactive since ${daysSinceInactive} days`,
+      `${this.getDryRunLogHeader(isDryRun)}Sending email to delete workspace ${
+        dataSource.workspaceId
+      } inactive since ${daysSinceInactive} days`,
    );

+    if (isDryRun) {
+      return;
+    }
+
    const emailData = {
-      daysSinceDead: daysSinceInactive - this.inactiveDaysBeforeDelete,
+      daysSinceInactive: daysSinceInactive,
      workspaceId: `${dataSource.workspaceId}`,
    };
-    const emailTemplate = DeleteInactiveWorkspaceEmail(emailData);
-    const html = render(emailTemplate, {
-      pretty: true,
-    });

-    const text = `Workspace '${dataSource.workspaceId}' should be deleted as inactive since ${daysSinceInactive} days`;
-
-    await this.emailService.send({
-      to: this.environmentService.getEmailSystemAddress(),
-      from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
-      subject: 'Action Needed to Delete Workspace',
-      html,
-      text,
-    });
+    this.workspacesToDelete.push(emailData);
  }

  async processWorkspace(
    dataSource: DataSourceEntity,
    objectsMetadata: ObjectMetadataEntity[],
+    isDryRun: boolean,
  ): Promise<void> {
-    const mostRecentUpdatedAt = await this.getmostRecentUpdatedAt(
+    const mostRecentUpdatedAt = await this.getMostRecentUpdatedAt(
      dataSource,
      objectsMetadata,
    );
@ -176,9 +187,9 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
    );

    if (daysSinceInactive > this.inactiveDaysBeforeDelete) {
-      await this.deleteWorkspace(dataSource, daysSinceInactive);
+      await this.deleteWorkspace(dataSource, daysSinceInactive, isDryRun);
    } else if (daysSinceInactive > this.inactiveDaysBeforeEmail) {
-      await this.warnWorkspaceUsers(dataSource, daysSinceInactive);
+      await this.warnWorkspaceUsers(dataSource, daysSinceInactive, isDryRun);
    }
  }

@ -196,8 +207,47 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
    );
  }

-  async handle(): Promise<void> {
-    this.logger.log('Job running...');
+  getDryRunLogHeader(isDryRun: boolean): string {
+    return isDryRun ? 'Dry-run mode: ' : '';
+  }
+
+  chunkArray(array: any[], chunkSize = 6): any[][] {
+    const chunkedArray: any[][] = [];
+    let index = 0;
+
+    while (index < array.length) {
+      chunkedArray.push(array.slice(index, index + chunkSize));
+      index += chunkSize;
+    }
+
+    return chunkedArray;
+  }
+
+  async sendDeleteWorkspaceEmail(isDryRun: boolean): Promise<void> {
+    if (isDryRun || this.workspacesToDelete.length === 0) {
+      return;
+    }
+    const emailTemplate = DeleteInactiveWorkspaceEmail(this.workspacesToDelete);
+    const html = render(emailTemplate, {
+      pretty: true,
+    });
+    const text = render(emailTemplate, {
+      plainText: true,
+    });
+
+    await this.emailService.send({
+      to: this.environmentService.getEmailSystemAddress(),
+      from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
+      subject: 'Action Needed to Delete Workspaces',
+      html,
+      text,
+    });
+  }
+
+  async handle(data: CleanInactiveWorkspacesCommandOptions): Promise<void> {
+    const isDryRun = data.dryRun || false;
+
+    this.logger.log(`${this.getDryRunLogHeader(isDryRun)}Job running...`);
    if (!this.inactiveDaysBeforeDelete && !this.inactiveDaysBeforeEmail) {
      this.logger.log(
        `'WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION' and 'WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION' environment variables not set, please check this doc for more info: https://docs.twenty.com/start/self-hosting/environment-variables`,
@ -205,20 +255,46 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {

      return;
    }
+
    const dataSources =
      await this.dataSourceService.getManyDataSourceMetadata();

-    const objectsMetadata = await this.objectMetadataService.findMany();
+    const dataSourcesChunks = this.chunkArray(dataSources);

-    for (const dataSource of dataSources) {
-      if (!(await this.isWorkspaceCleanable(dataSource))) {
-        continue;
+    this.logger.log(
+      `${this.getDryRunLogHeader(isDryRun)}On ${
+        dataSources.length
+      } workspaces divided in ${dataSourcesChunks.length} chunks...`,
+    );
+
+    for (const dataSourcesChunk of dataSourcesChunks) {
+      const objectsMetadata = await this.objectMetadataService.findMany({
+        where: {
+          dataSourceId: In(dataSourcesChunk.map((dataSource) => dataSource.id)),
+        },
+      });
+
+      for (const dataSource of dataSourcesChunk) {
+        if (!(await this.isWorkspaceCleanable(dataSource))) {
+          this.logger.log(
+            `${this.getDryRunLogHeader(isDryRun)}Workspace ${
+              dataSource.workspaceId
+            } not cleanable`,
+          );
+          continue;
+        }
+
+        this.logger.log(
+          `${this.getDryRunLogHeader(isDryRun)}Cleaning Workspace ${
+            dataSource.workspaceId
+          }`,
+        );
+        await this.processWorkspace(dataSource, objectsMetadata, isDryRun);
      }
-
-      this.logger.log(`Cleaning Workspace ${dataSource.workspaceId}`);
-      await this.processWorkspace(dataSource, objectsMetadata);
    }

-    this.logger.log('job done!');
+    await this.sendDeleteWorkspaceEmail(isDryRun);
+
+    this.logger.log(`${this.getDryRunLogHeader(isDryRun)}job done!`);
  }
 }
--- a/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/clean-inactive-workspaces.command.ts
+++ b/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/clean-inactive-workspaces.command.ts
@ -1,11 +1,15 @@
 import { Inject } from '@nestjs/common';

-import { Command, CommandRunner } from 'nest-commander';
+import { Command, CommandRunner, Option } from 'nest-commander';

 import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
 import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';

+export type CleanInactiveWorkspacesCommandOptions = {
+  dryRun: boolean;
+};
+
@Command({
  name: 'clean-inactive-workspaces',
  description: 'Clean inactive workspaces',
@ -18,10 +22,22 @@ export class CleanInactiveWorkspacesCommand extends CommandRunner {
    super();
  }

-  async run(): Promise<void> {
-    await this.messageQueueService.add<any>(
+  @Option({
+    flags: '-d, --dry-run [dry run]',
+    description: 'Dry run: Log cleaning actions without executing them.',
+    required: false,
+  })
+  dryRun(value: string): boolean {
+    return Boolean(value);
+  }
+
+  async run(
+    _passedParam: string[],
+    options: CleanInactiveWorkspacesCommandOptions,
+  ): Promise<void> {
+    await this.messageQueueService.add<CleanInactiveWorkspacesCommandOptions>(
      CleanInactiveWorkspaceJob.name,
-      {},
+      options,
      { retryLimit: 3 },
    );
  }
--- a/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/start-clean-inactive-workspaces.cron.command.ts
+++ b/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/start-clean-inactive-workspaces.cron.command.ts
@ -8,7 +8,7 @@ import { cleanInactiveWorkspaceCronPattern } from 'src/workspace/cron/clean-inac
 import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';

@Command({
-  name: 'clean-inactive-workspace:cron:start',
+  name: 'clean-inactive-workspaces:cron:start',
  description: 'Starts a cron job to clean inactive workspaces',
 })
 export class StartCleanInactiveWorkspacesCronCommand extends CommandRunner {
--- a/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/stop-clean-inactive-workspaces.cron.command.ts
+++ b/packages/twenty-server/src/workspace/cron/clean-inactive-workspaces/commands/stop-clean-inactive-workspaces.cron.command.ts
@ -8,7 +8,7 @@ import { cleanInactiveWorkspaceCronPattern } from 'src/workspace/cron/clean-inac
 import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';

@Command({
-  name: 'clean-inactive-workspace:cron:stop',
+  name: 'clean-inactive-workspaces:cron:stop',
  description: 'Stops the clean inactive workspaces cron job',
 })
 export class StopCleanInactiveWorkspacesCronCommand extends CommandRunner {