Update clean inactive workspaces (#3600)

* Fix typo * Add dry-run option in clean inactive workspaces * Add logs * Chunk workspace metadata * Add BCC to clean workspace notification email * Send workspace to delete ids in one email * Update example * Update function naming
2024-01-24 12:51:42 +01:00
parent f48814f6d9
commit b991790f62
8 changed files with 157 additions and 54 deletions
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';

 import { render } from '@react-email/render';
-import { Repository } from 'typeorm';
+import { In, Repository } from 'typeorm';
 import {
  CleanInactiveWorkspaceEmail,
  DeleteInactiveWorkspaceEmail,
@@ -23,14 +23,23 @@ import {
 } from 'src/core/feature-flag/feature-flag.entity';
 import { ObjectMetadataEntity } from 'src/metadata/object-metadata/object-metadata.entity';
 import { computeObjectTargetTable } from 'src/workspace/utils/compute-object-target-table.util';
+import { CleanInactiveWorkspacesCommandOptions } from 'src/workspace/cron/clean-inactive-workspaces/commands/clean-inactive-workspaces.command';

 const MILLISECONDS_IN_ONE_DAY = 1000 * 3600 * 24;

+type WorkspaceToDeleteData = {
+  workspaceId: string;
+  daysSinceInactive: number;
+};
+
@Injectable()
-export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
+export class CleanInactiveWorkspaceJob
+  implements MessageQueueJob<CleanInactiveWorkspacesCommandOptions>
+{
  private readonly logger = new Logger(CleanInactiveWorkspaceJob.name);
  private readonly inactiveDaysBeforeDelete;
  private readonly inactiveDaysBeforeEmail;
+  private workspacesToDelete: WorkspaceToDeleteData[] = [];

  constructor(
    private readonly dataSourceService: DataSourceService,
@@ -48,7 +57,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
      this.environmentService.getInactiveDaysBeforeEmail();
  }

-  async getmostRecentUpdatedAt(
+  async getMostRecentUpdatedAt(
    dataSource: DataSourceEntity,
    objectsMetadata: ObjectMetadataEntity[],
  ): Promise<Date> {
@@ -89,6 +98,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
  async warnWorkspaceUsers(
    dataSource: DataSourceEntity,
    daysSinceInactive: number,
+    isDryRun: boolean,
  ) {
    const workspaceMembers =
      await this.userService.loadWorkspaceMembers(dataSource);
@@ -103,13 +113,17 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
    )?.[0].displayName;

    this.logger.log(
-      `Sending workspace ${
+      `${this.getDryRunLogHeader(isDryRun)}Sending workspace ${
        dataSource.workspaceId
      } inactive since ${daysSinceInactive} days emails to users ['${workspaceMembers
        .map((workspaceUser) => workspaceUser.email)
        .join(', ')}']`,
    );

+    if (isDryRun) {
+      return;
+    }
+
    workspaceMembers.forEach((workspaceMember) => {
      const emailData = {
        daysLeft: this.inactiveDaysBeforeDelete - daysSinceInactive,
@@ -126,6 +140,7 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {

      this.emailService.send({
        to: workspaceMember.email,
+        bcc: this.environmentService.getEmailSystemAddress(),
        from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
        subject: 'Action Needed to Prevent Workspace Deletion',
        html,
@@ -137,36 +152,32 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
  async deleteWorkspace(
    dataSource: DataSourceEntity,
    daysSinceInactive: number,
+    isDryRun: boolean,
  ): Promise<void> {
    this.logger.log(
-      `Sending email to delete workspace ${dataSource.workspaceId} inactive since ${daysSinceInactive} days`,
+      `${this.getDryRunLogHeader(isDryRun)}Sending email to delete workspace ${
+        dataSource.workspaceId
+      } inactive since ${daysSinceInactive} days`,
    );

+    if (isDryRun) {
+      return;
+    }
+
    const emailData = {
-      daysSinceDead: daysSinceInactive - this.inactiveDaysBeforeDelete,
+      daysSinceInactive: daysSinceInactive,
      workspaceId: `${dataSource.workspaceId}`,
    };
-    const emailTemplate = DeleteInactiveWorkspaceEmail(emailData);
-    const html = render(emailTemplate, {
-      pretty: true,
-    });

-    const text = `Workspace '${dataSource.workspaceId}' should be deleted as inactive since ${daysSinceInactive} days`;
-
-    await this.emailService.send({
-      to: this.environmentService.getEmailSystemAddress(),
-      from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
-      subject: 'Action Needed to Delete Workspace',
-      html,
-      text,
-    });
+    this.workspacesToDelete.push(emailData);
  }

  async processWorkspace(
    dataSource: DataSourceEntity,
    objectsMetadata: ObjectMetadataEntity[],
+    isDryRun: boolean,
  ): Promise<void> {
-    const mostRecentUpdatedAt = await this.getmostRecentUpdatedAt(
+    const mostRecentUpdatedAt = await this.getMostRecentUpdatedAt(
      dataSource,
      objectsMetadata,
    );
@@ -176,9 +187,9 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
    );

    if (daysSinceInactive > this.inactiveDaysBeforeDelete) {
-      await this.deleteWorkspace(dataSource, daysSinceInactive);
+      await this.deleteWorkspace(dataSource, daysSinceInactive, isDryRun);
    } else if (daysSinceInactive > this.inactiveDaysBeforeEmail) {
-      await this.warnWorkspaceUsers(dataSource, daysSinceInactive);
+      await this.warnWorkspaceUsers(dataSource, daysSinceInactive, isDryRun);
    }
  }

@@ -196,8 +207,47 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {
    );
  }

-  async handle(): Promise<void> {
-    this.logger.log('Job running...');
+  getDryRunLogHeader(isDryRun: boolean): string {
+    return isDryRun ? 'Dry-run mode: ' : '';
+  }
+
+  chunkArray(array: any[], chunkSize = 6): any[][] {
+    const chunkedArray: any[][] = [];
+    let index = 0;
+
+    while (index < array.length) {
+      chunkedArray.push(array.slice(index, index + chunkSize));
+      index += chunkSize;
+    }
+
+    return chunkedArray;
+  }
+
+  async sendDeleteWorkspaceEmail(isDryRun: boolean): Promise<void> {
+    if (isDryRun || this.workspacesToDelete.length === 0) {
+      return;
+    }
+    const emailTemplate = DeleteInactiveWorkspaceEmail(this.workspacesToDelete);
+    const html = render(emailTemplate, {
+      pretty: true,
+    });
+    const text = render(emailTemplate, {
+      plainText: true,
+    });
+
+    await this.emailService.send({
+      to: this.environmentService.getEmailSystemAddress(),
+      from: `${this.environmentService.getEmailFromName()} <${this.environmentService.getEmailFromAddress()}>`,
+      subject: 'Action Needed to Delete Workspaces',
+      html,
+      text,
+    });
+  }
+
+  async handle(data: CleanInactiveWorkspacesCommandOptions): Promise<void> {
+    const isDryRun = data.dryRun || false;
+
+    this.logger.log(`${this.getDryRunLogHeader(isDryRun)}Job running...`);
    if (!this.inactiveDaysBeforeDelete && !this.inactiveDaysBeforeEmail) {
      this.logger.log(
        `'WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION' and 'WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION' environment variables not set, please check this doc for more info: https://docs.twenty.com/start/self-hosting/environment-variables`,
@@ -205,20 +255,46 @@ export class CleanInactiveWorkspaceJob implements MessageQueueJob<undefined> {

      return;
    }
+
    const dataSources =
      await this.dataSourceService.getManyDataSourceMetadata();

-    const objectsMetadata = await this.objectMetadataService.findMany();
+    const dataSourcesChunks = this.chunkArray(dataSources);

-    for (const dataSource of dataSources) {
-      if (!(await this.isWorkspaceCleanable(dataSource))) {
-        continue;
+    this.logger.log(
+      `${this.getDryRunLogHeader(isDryRun)}On ${
+        dataSources.length
+      } workspaces divided in ${dataSourcesChunks.length} chunks...`,
+    );
+
+    for (const dataSourcesChunk of dataSourcesChunks) {
+      const objectsMetadata = await this.objectMetadataService.findMany({
+        where: {
+          dataSourceId: In(dataSourcesChunk.map((dataSource) => dataSource.id)),
+        },
+      });
+
+      for (const dataSource of dataSourcesChunk) {
+        if (!(await this.isWorkspaceCleanable(dataSource))) {
+          this.logger.log(
+            `${this.getDryRunLogHeader(isDryRun)}Workspace ${
+              dataSource.workspaceId
+            } not cleanable`,
+          );
+          continue;
+        }
+
+        this.logger.log(
+          `${this.getDryRunLogHeader(isDryRun)}Cleaning Workspace ${
+            dataSource.workspaceId
+          }`,
+        );
+        await this.processWorkspace(dataSource, objectsMetadata, isDryRun);
      }
-
-      this.logger.log(`Cleaning Workspace ${dataSource.workspaceId}`);
-      await this.processWorkspace(dataSource, objectsMetadata);
    }

-    this.logger.log('job done!');
+    await this.sendDeleteWorkspaceEmail(isDryRun);
+
+    this.logger.log(`${this.getDryRunLogHeader(isDryRun)}job done!`);
  }
 }
@@ -1,11 +1,15 @@
 import { Inject } from '@nestjs/common';

-import { Command, CommandRunner } from 'nest-commander';
+import { Command, CommandRunner, Option } from 'nest-commander';

 import { MessageQueue } from 'src/integrations/message-queue/message-queue.constants';
 import { MessageQueueService } from 'src/integrations/message-queue/services/message-queue.service';
 import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';

+export type CleanInactiveWorkspacesCommandOptions = {
+  dryRun: boolean;
+};
+
@Command({
  name: 'clean-inactive-workspaces',
  description: 'Clean inactive workspaces',
@@ -18,10 +22,22 @@ export class CleanInactiveWorkspacesCommand extends CommandRunner {
    super();
  }

-  async run(): Promise<void> {
-    await this.messageQueueService.add<any>(
+  @Option({
+    flags: '-d, --dry-run [dry run]',
+    description: 'Dry run: Log cleaning actions without executing them.',
+    required: false,
+  })
+  dryRun(value: string): boolean {
+    return Boolean(value);
+  }
+
+  async run(
+    _passedParam: string[],
+    options: CleanInactiveWorkspacesCommandOptions,
+  ): Promise<void> {
+    await this.messageQueueService.add<CleanInactiveWorkspacesCommandOptions>(
      CleanInactiveWorkspaceJob.name,
-      {},
+      options,
      { retryLimit: 3 },
    );
  }
@@ -8,7 +8,7 @@ import { cleanInactiveWorkspaceCronPattern } from 'src/workspace/cron/clean-inac
 import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';

@Command({
-  name: 'clean-inactive-workspace:cron:start',
+  name: 'clean-inactive-workspaces:cron:start',
  description: 'Starts a cron job to clean inactive workspaces',
 })
 export class StartCleanInactiveWorkspacesCronCommand extends CommandRunner {
@@ -8,7 +8,7 @@ import { cleanInactiveWorkspaceCronPattern } from 'src/workspace/cron/clean-inac
 import { CleanInactiveWorkspaceJob } from 'src/workspace/cron/clean-inactive-workspaces/clean-inactive-workspace.job';

@Command({
-  name: 'clean-inactive-workspace:cron:stop',
+  name: 'clean-inactive-workspaces:cron:stop',
  description: 'Stops the clean inactive workspaces cron job',
 })
 export class StopCleanInactiveWorkspacesCronCommand extends CommandRunner {