0.2.0 cleaning script (#2403)

* Update cleaning script to run on old schema

* Add boundaries parameter

* Stop requesting data for each workspace/table

* Stop checking same as seed if not requested

* Minor update

* Minor update

* Minor update

* Minor update

* Minor update

* Simplify result

* Simplify result

* Simplify result

* Delete updates

* Fix issues

* Update logs

* Remove throw when schema does not exist

* Remove missing table in old schema

* Remove boundaries parameter

* Remove useless trycatch

server/src/database/commands/clean-inactive-workspaces.command.ts

@@ -25,17 +25,19 @@ interface ActivityReport {
   displayName: string;
   maxUpdatedAt: string;
   inactiveDays: number;
-}
+  sameAsSeed: boolean;
-
-interface SameAsSeedWorkspace {
-  displayName: string;
 }
 
 interface DataCleanResults {
-  activityReport: { [key: string]: ActivityReport };
+  [key: string]: ActivityReport;
-  sameAsSeedWorkspaces: { [key: string]: SameAsSeedWorkspace };
 }
 
+const formattedPipelineStagesSeed = pipelineStagesSeed.map((pipelineStage) => {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const { position, ...rest } = pipelineStage;
+  return rest;
+});
+
 @Command({
   name: 'workspaces:clean-inactive',
   description: 'Clean inactive workspaces from the public database schema',
@@ -83,7 +85,7 @@ export class DataCleanInactiveCommand extends CommandRunner {
     return Boolean(val);
   }
 
-  // We look for public tables which contains workspaceId and updatedAt columns
+  // We look for public tables which contain workspaceId and updatedAt columns and exist in production database
   getRelevantTables() {
     return Object.keys(this.prismaService.client).filter(
       (name) =>
@@ -96,36 +98,58 @@ export class DataCleanInactiveCommand extends CommandRunner {
     );
   }
 
-  async getTableMaxUpdatedAt(table, workspace) {
+  async getMaxUpdatedAtForAllWorkspaces(tables, workspaces) {
-    return await this.prismaService.client[table].aggregate({
+    const result = {};
-      _max: { updatedAt: true },
+    for (const table of tables) {
-      where: { workspaceId: { equals: workspace.id } },
+      result[table] = {};
-    });
+      const groupByWorkspaces = await this.prismaService.client[table].groupBy({
+        by: ['workspaceId'],
+        _max: { updatedAt: true },
+        where: {
+          workspaceId: { in: workspaces.map((workspace) => workspace.id) },
+        },
+      });
+      for (const groupByWorkspace of groupByWorkspaces) {
+        result[table][groupByWorkspace.workspaceId] =
+          groupByWorkspace._max.updatedAt;
+      }
+    }
+    return result;
   }
 
-  async addMaxUpdatedAtToWorkspaces(result, workspace, table) {
+  async addMaxUpdatedAtToWorkspaces(
-    const newUpdatedAt = await this.getTableMaxUpdatedAt(table, workspace);
+    result,
-    if (!result.activityReport[workspace.id]) {
+    workspace,
-      result.activityReport[workspace.id] = {
+    table,
+    maxUpdatedAtForAllWorkspaces,
+  ) {
+    const newUpdatedAt = maxUpdatedAtForAllWorkspaces[table][workspace.id];
+    if (!result[workspace.id]) {
+      result[workspace.id] = {
         displayName: workspace.displayName,
         maxUpdatedAt: null,
       };
     }
     if (
       newUpdatedAt &&
-      newUpdatedAt._max.updatedAt &&
+      new Date(result[workspace.id].maxUpdatedAt) < new Date(newUpdatedAt)
-      new Date(result.activityReport[workspace.id].maxUpdatedAt) <
-        new Date(newUpdatedAt._max.updatedAt)
     ) {
-      result.activityReport[workspace.id].maxUpdatedAt =
+      result[workspace.id].maxUpdatedAt = newUpdatedAt;
-        newUpdatedAt._max.updatedAt;
     }
   }
-
+  async getSeedTableData(workspaces) {
-  async detectWorkspacesWithSeedDataOnly(result, workspace) {
+    const where = {
+      workspaceId: { in: workspaces.map((workspace) => workspace.id) },
+    };
     const companies = await this.prismaService.client.company.findMany({
-      select: { name: true, domainName: true, address: true, employees: true },
+      select: {
-      where: { workspaceId: { equals: workspace.id } },
+        name: true,
+        domainName: true,
+        address: true,
+        employees: true,
+        workspaceId: true,
+      },
+      where,
     });
     const people = await this.prismaService.client.person.findMany({
       select: {
@@ -134,121 +158,178 @@ export class DataCleanInactiveCommand extends CommandRunner {
         city: true,
         email: true,
         avatarUrl: true,
+        workspaceId: true,
       },
-      where: { workspaceId: { equals: workspace.id } },
+      where,
     });
     const pipelineStages =
       await this.prismaService.client.pipelineStage.findMany({
         select: {
           name: true,
           color: true,
-          position: true,
           type: true,
+          workspaceId: true,
         },
-        where: { workspaceId: { equals: workspace.id } },
+        where,
       });
     const pipelines = await this.prismaService.client.pipeline.findMany({
       select: {
         name: true,
         icon: true,
         pipelineProgressableType: true,
+        workspaceId: true,
       },
-      where: { workspaceId: { equals: workspace.id } },
+      where,
     });
+    return {
+      companies,
+      people,
+      pipelineStages,
+      pipelines,
+    };
+  }
+
+  async detectWorkspacesWithSeedDataOnly(result, workspace, seedTableData) {
+    const companies = seedTableData.companies.reduce((filtered, company) => {
+      if (company.workspaceId === workspace.id) {
+        delete company.workspaceId;
+        filtered.push(company);
+      }
+      return filtered;
+    }, []);
+    const people = seedTableData.people.reduce((filtered, person) => {
+      if (person.workspaceId === workspace.id) {
+        delete person.workspaceId;
+        filtered.push(person);
+      }
+      return filtered;
+    }, []);
+    const pipelineStages = seedTableData.pipelineStages.reduce(
+      (filtered, pipelineStage) => {
+        if (pipelineStage.workspaceId === workspace.id) {
+          delete pipelineStage.workspaceId;
+          filtered.push(pipelineStage);
+        }
+        return filtered;
+      },
+      [],
+    );
+    const pipelines = seedTableData.pipelines.reduce((filtered, pipeline) => {
+      if (pipeline.workspaceId === workspace.id) {
+        delete pipeline.workspaceId;
+        filtered.push(pipeline);
+      }
+      return filtered;
+    }, []);
     if (
       isEqual(people, peopleSeed) &&
       isEqual(companies, companiesSeed) &&
-      isEqual(pipelineStages, pipelineStagesSeed) &&
+      isEqual(pipelineStages, formattedPipelineStagesSeed) &&
       isEqual(pipelines, [pipelinesSeed])
     ) {
-      result.sameAsSeedWorkspaces[workspace.id] = {
+      result[workspace.id].sameAsSeed = true;
-        displayName: workspace.displayName,
+    } else {
-      };
+      {
+        result[workspace.id].sameAsSeed = false;
+      }
     }
   }
 
-  async findInactiveWorkspaces(result, options) {
+  async getWorkspaces(options) {
     const where = options.workspaceId
       ? { id: { equals: options.workspaceId } }
       : {};
-    const workspaces = await this.prismaService.client.workspace.findMany({
+    return await this.prismaService.client.workspace.findMany({
       where,
+      orderBy: [{ createdAt: 'asc' }],
     });
+  }
+
+  async findInactiveWorkspaces(workspaces, result) {
     const tables = this.getRelevantTables();
+    const maxUpdatedAtForAllWorkspaces =
+      await this.getMaxUpdatedAtForAllWorkspaces(tables, workspaces);
+    const seedTableData = await this.getSeedTableData(workspaces);
     for (const workspace of workspaces) {
-      await this.detectWorkspacesWithSeedDataOnly(result, workspace);
       for (const table of tables) {
-        await this.addMaxUpdatedAtToWorkspaces(result, workspace, table);
+        await this.addMaxUpdatedAtToWorkspaces(
+          result,
+          workspace,
+          table,
+          maxUpdatedAtForAllWorkspaces,
+        );
       }
+      await this.detectWorkspacesWithSeedDataOnly(
+        result,
+        workspace,
+        seedTableData,
+      );
     }
   }
 
   filterResults(result, options) {
-    for (const workspaceId in result.activityReport) {
+    for (const workspaceId in result) {
       const timeDifferenceInSeconds = Math.abs(
         new Date().getTime() -
-          new Date(result.activityReport[workspaceId].maxUpdatedAt).getTime(),
+          new Date(result[workspaceId].maxUpdatedAt).getTime(),
       );
       const timeDifferenceInDays = Math.ceil(
         timeDifferenceInSeconds / (1000 * 3600 * 24),
       );
-      if (timeDifferenceInDays < options.sameAsSeedDays) {
+      if (
-        delete result.sameAsSeedWorkspaces[workspaceId];
+        timeDifferenceInDays < options.days &&
-      }
+        (!result[workspaceId].sameAsSeed ||
-      if (timeDifferenceInDays < options.days) {
+          timeDifferenceInDays < options.sameAsSeedDays)
-        delete result.activityReport[workspaceId];
+      ) {
+        delete result[workspaceId];
       } else {
-        result.activityReport[workspaceId].inactiveDays = timeDifferenceInDays;
+        result[workspaceId].inactiveDays = timeDifferenceInDays;
       }
     }
   }
 
-  async delete(result) {
+  async delete(result, options) {
-    if (Object.keys(result.activityReport).length) {
+    const workspaceCount = Object.keys(result).length;
-      console.log('Deleting inactive workspaces');
+    if (workspaceCount) {
+      console.log(
+        `Deleting \x1b[36m${workspaceCount}\x1b[0m inactive since \x1b[36m${options.days} days\x1b[0m or same as seed since \x1b[36m${options.sameAsSeedDays} days\x1b[0m workspaces`,
+      );
     }
-    for (const workspaceId in result.activityReport) {
+    let count = 1;
+    for (const workspaceId in result) {
       process.stdout.write(`- deleting ${workspaceId} ...`);
       await this.workspaceService.deleteWorkspace({
         workspaceId,
       });
-      console.log(' done!');
+      console.log(
-    }
+        ` done! ....... ${Math.floor((100 * count) / workspaceCount)}%`,
-    if (Object.keys(result.sameAsSeedWorkspaces).length) {
+      );
-      console.log('Deleting same as Seed workspaces');
+      count += 1;
-    }
-    for (const workspaceId in result.sameAsSeedWorkspaces) {
-      process.stdout.write(`- deleting ${workspaceId} ...`);
-      await this.workspaceService.deleteWorkspace({
-        workspaceId,
-      });
-      console.log(' done!');
     }
   }
 
-  displayResults(result) {
+  displayResults(result, totalWorkspacesCount) {
-    const workspacesToDelete = new Set();
-    for (const workspaceId in result.activityReport) {
-      workspacesToDelete.add(workspaceId);
-    }
-    for (const workspaceId in result.sameAsSeedWorkspaces) {
-      workspacesToDelete.add(workspaceId);
-    }
-    console.log(`${workspacesToDelete.size} workspace(s) will be deleted:`);
     console.log(result);
+    console.log(
+      `${
+        Object.keys(result).length
+      } out of ${totalWorkspacesCount} workspace(s) checked (${Math.floor(
+        (100 * Object.keys(result).length) / totalWorkspacesCount,
+      )}%) will be deleted`,
+    );
   }
 
   async run(
     _passedParam: string[],
     options: DataCleanInactiveOptions,
   ): Promise<void> {
-    const result: DataCleanResults = {
+    const result: DataCleanResults = {};
-      activityReport: {},
+    const workspaces = await this.getWorkspaces(options);
-      sameAsSeedWorkspaces: {},
+    const totalWorkspacesCount = workspaces.length;
-    };
+    console.log(totalWorkspacesCount, 'workspace(s) to analyse');
-    await this.findInactiveWorkspaces(result, options);
+    await this.findInactiveWorkspaces(workspaces, result);
     this.filterResults(result, options);
-    this.displayResults(result);
+    this.displayResults(result, totalWorkspacesCount);
     if (!options.dryRun) {
       options = await this.inquiererService.ask('confirm', options);
       if (!options.confirmation) {
@@ -257,7 +338,7 @@ export class DataCleanInactiveCommand extends CommandRunner {
       }
     }
     if (!options.dryRun) {
-      await this.delete(result);
+      await this.delete(result, options);
     }
   }
 }

server/src/metadata/data-source/data-source.service.ts

@@ -127,7 +127,8 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
     const schemaAlreadyExists = await queryRunner.hasSchema(schemaName);
 
     if (!schemaAlreadyExists) {
-      throw new Error(`Schema ${schemaName} does not exist`);
+      await queryRunner.release();
+      return;
     }
 
     await queryRunner.dropSchema(schemaName, true, true);

server/src/metadata/field-metadata/services/field-metadata.service.ts

@@ -103,4 +103,8 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadata> {
 
     return createdFieldMetadata;
   }
+
+  public async deleteFieldsMetadata(workspaceId: string) {
+    await this.fieldMetadataRepository.delete({ workspaceId });
+  }
 }

server/src/metadata/metadata.datasource.ts

@@ -15,7 +15,7 @@ export const typeORMMetadataModuleOptions: TypeOrmModuleOptions = {
   schema: 'metadata',
   entities: [__dirname + '/**/*.entity{.ts,.js}'],
   synchronize: false,
-  migrationsRun: true,
+  migrationsRun: false,
   migrationsTableName: '_typeorm_migrations',
   migrations: [__dirname + '/migrations/*{.ts,.js}'],
 };

server/src/metadata/object-metadata/services/object-metadata.service.ts

@@ -131,7 +131,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadata> {
     );
   }
 
-  public async deleteObjectsAndFieldsMetadata(workspaceId: string) {
+  public async deleteObjectsMetadata(workspaceId: string) {
     await this.objectMetadataRepository.delete({ workspaceId });
   }
 }

server/src/metadata/tenant-initialisation/tenant-initialisation.module.ts

@@ -5,6 +5,7 @@ import { MigrationRunnerModule } from 'src/metadata/migration-runner/migration-r
 import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module';
 import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
 import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
+import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module';
 
 import { TenantInitialisationService } from './tenant-initialisation.service';
 
@@ -14,6 +15,7 @@ import { TenantInitialisationService } from './tenant-initialisation.service';
     TenantMigrationModule,
     MigrationRunnerModule,
     ObjectMetadataModule,
+    FieldMetadataModule,
     DataSourceMetadataModule,
   ],
   exports: [TenantInitialisationService],

server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts

@@ -6,6 +6,7 @@ import { DataSourceService } from 'src/metadata/data-source/data-source.service'
 import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
 import { ObjectMetadataService } from 'src/metadata/object-metadata/services/object-metadata.service';
 import { DataSourceMetadata } from 'src/metadata/data-source-metadata/data-source-metadata.entity';
+import { FieldMetadataService } from 'src/metadata/field-metadata/services/field-metadata.service';
 
 import { standardObjectsPrefillData } from './standard-objects-prefill-data/standard-objects-prefill-data';
 
@@ -16,6 +17,7 @@ export class TenantInitialisationService {
     private readonly tenantMigrationService: TenantMigrationService,
     private readonly migrationRunnerService: MigrationRunnerService,
     private readonly objectMetadataService: ObjectMetadataService,
+    private readonly fieldMetadataService: FieldMetadataService,
     private readonly dataSourceMetadataService: DataSourceMetadataService,
   ) {}
 
@@ -78,9 +80,8 @@ export class TenantInitialisationService {
 
   public async delete(workspaceId: string): Promise<void> {
     // Delete data from metadata tables
-    await this.objectMetadataService.deleteObjectsAndFieldsMetadata(
+    await this.fieldMetadataService.deleteFieldsMetadata(workspaceId);
-      workspaceId,
+    await this.objectMetadataService.deleteObjectsMetadata(workspaceId);
-    );
     await this.tenantMigrationService.delete(workspaceId);
     await this.dataSourceMetadataService.delete(workspaceId);
     // Delete schema