admin/twenty
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

0.2.0 cleaning script (#2403)

Browse Source 
* Update cleaning script to run on old schema

* Add boundaries parameter

* Stop requesting data for each workspace/table

* Stop checking same as seed if not requested

* Minor update

* Minor update

* Minor update

* Minor update

* Minor update

* Simplify result

* Simplify result

* Simplify result

* Delete updates

* Fix issues

* Update logs

* Remove throw when schema does not exist

* Remove missing table in old schema

* Remove boundaries parameter

* Remove useless trycatch
This commit is contained in:
martmull
2023-11-09 12:18:09 +01:00
committed by GitHub
parent 28779f0fb8
commit fe20be8487
7 changed files with 171 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
server/src/database/commands/clean-inactive-workspaces.command.ts
View File

@ -25,17 +25,19 @@ interface ActivityReport {
displayName: string;
maxUpdatedAt: string;
inactiveDays: number;
}
interface SameAsSeedWorkspace {
displayName: string;
sameAsSeed: boolean;
}
interface DataCleanResults {
activityReport: { [key: string]: ActivityReport };
sameAsSeedWorkspaces: { [key: string]: SameAsSeedWorkspace };
[key: string]: ActivityReport;
}
const formattedPipelineStagesSeed = pipelineStagesSeed.map((pipelineStage) => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { position, ...rest } = pipelineStage;
return rest;
});
@Command({
name: 'workspaces:clean-inactive',
description: 'Clean inactive workspaces from the public database schema',
@ -83,7 +85,7 @@ export class DataCleanInactiveCommand extends CommandRunner {
return Boolean(val);
}
// We look for public tables which contains workspaceId and updatedAt columns
// We look for public tables which contain workspaceId and updatedAt columns and exist in production database
getRelevantTables() {
return Object.keys(this.prismaService.client).filter(
(name) =>
@ -96,36 +98,58 @@ export class DataCleanInactiveCommand extends CommandRunner {
);
}
async getTableMaxUpdatedAt(table, workspace) {
return await this.prismaService.client[table].aggregate({
_max: { updatedAt: true },
where: { workspaceId: { equals: workspace.id } },
});
async getMaxUpdatedAtForAllWorkspaces(tables, workspaces) {
const result = {};
for (const table of tables) {
result[table] = {};
const groupByWorkspaces = await this.prismaService.client[table].groupBy({
by: ['workspaceId'],
_max: { updatedAt: true },
where: {
workspaceId: { in: workspaces.map((workspace) => workspace.id) },
},
});
for (const groupByWorkspace of groupByWorkspaces) {
result[table][groupByWorkspace.workspaceId] =
groupByWorkspace._max.updatedAt;
}
}
return result;
}
async addMaxUpdatedAtToWorkspaces(result, workspace, table) {
const newUpdatedAt = await this.getTableMaxUpdatedAt(table, workspace);
if (!result.activityReport[workspace.id]) {
result.activityReport[workspace.id] = {
async addMaxUpdatedAtToWorkspaces(
result,
workspace,
table,
maxUpdatedAtForAllWorkspaces,
) {
const newUpdatedAt = maxUpdatedAtForAllWorkspaces[table][workspace.id];
if (!result[workspace.id]) {
result[workspace.id] = {
displayName: workspace.displayName,
maxUpdatedAt: null,
};
}
if (
newUpdatedAt &&
newUpdatedAt._max.updatedAt &&
new Date(result.activityReport[workspace.id].maxUpdatedAt) <
new Date(newUpdatedAt._max.updatedAt)
new Date(result[workspace.id].maxUpdatedAt) < new Date(newUpdatedAt)
) {
result.activityReport[workspace.id].maxUpdatedAt =
newUpdatedAt._max.updatedAt;
result[workspace.id].maxUpdatedAt = newUpdatedAt;
}
}
async detectWorkspacesWithSeedDataOnly(result, workspace) {
async getSeedTableData(workspaces) {
const where = {
workspaceId: { in: workspaces.map((workspace) => workspace.id) },
};
const companies = await this.prismaService.client.company.findMany({
select: { name: true, domainName: true, address: true, employees: true },
where: { workspaceId: { equals: workspace.id } },
select: {
name: true,
domainName: true,
address: true,
employees: true,
workspaceId: true,
},
where,
});
const people = await this.prismaService.client.person.findMany({
select: {
@ -134,121 +158,178 @@ export class DataCleanInactiveCommand extends CommandRunner {
city: true,
email: true,
avatarUrl: true,
workspaceId: true,
},
where: { workspaceId: { equals: workspace.id } },
where,
});
const pipelineStages =
await this.prismaService.client.pipelineStage.findMany({
select: {
name: true,
color: true,
position: true,
type: true,
workspaceId: true,
},
where: { workspaceId: { equals: workspace.id } },
where,
});
const pipelines = await this.prismaService.client.pipeline.findMany({
select: {
name: true,
icon: true,
pipelineProgressableType: true,
workspaceId: true,
},
where: { workspaceId: { equals: workspace.id } },
where,
});
return {
companies,
people,
pipelineStages,
pipelines,
};
}
async detectWorkspacesWithSeedDataOnly(result, workspace, seedTableData) {
const companies = seedTableData.companies.reduce((filtered, company) => {
if (company.workspaceId === workspace.id) {
delete company.workspaceId;
filtered.push(company);
}
return filtered;
}, []);
const people = seedTableData.people.reduce((filtered, person) => {
if (person.workspaceId === workspace.id) {
delete person.workspaceId;
filtered.push(person);
}
return filtered;
}, []);
const pipelineStages = seedTableData.pipelineStages.reduce(
(filtered, pipelineStage) => {
if (pipelineStage.workspaceId === workspace.id) {
delete pipelineStage.workspaceId;
filtered.push(pipelineStage);
}
return filtered;
},
[],
);
const pipelines = seedTableData.pipelines.reduce((filtered, pipeline) => {
if (pipeline.workspaceId === workspace.id) {
delete pipeline.workspaceId;
filtered.push(pipeline);
}
return filtered;
}, []);
if (
isEqual(people, peopleSeed) &&
isEqual(companies, companiesSeed) &&
isEqual(pipelineStages, pipelineStagesSeed) &&
isEqual(pipelineStages, formattedPipelineStagesSeed) &&
isEqual(pipelines, [pipelinesSeed])
) {
result.sameAsSeedWorkspaces[workspace.id] = {
displayName: workspace.displayName,
};
result[workspace.id].sameAsSeed = true;
} else {
{
result[workspace.id].sameAsSeed = false;
}
}
}
async findInactiveWorkspaces(result, options) {
async getWorkspaces(options) {
const where = options.workspaceId
? { id: { equals: options.workspaceId } }
: {};
const workspaces = await this.prismaService.client.workspace.findMany({
return await this.prismaService.client.workspace.findMany({
where,
orderBy: [{ createdAt: 'asc' }],
});
}
async findInactiveWorkspaces(workspaces, result) {
const tables = this.getRelevantTables();
const maxUpdatedAtForAllWorkspaces =
await this.getMaxUpdatedAtForAllWorkspaces(tables, workspaces);
const seedTableData = await this.getSeedTableData(workspaces);
for (const workspace of workspaces) {
await this.detectWorkspacesWithSeedDataOnly(result, workspace);
for (const table of tables) {
await this.addMaxUpdatedAtToWorkspaces(result, workspace, table);
await this.addMaxUpdatedAtToWorkspaces(
result,
workspace,
table,
maxUpdatedAtForAllWorkspaces,
);
}
await this.detectWorkspacesWithSeedDataOnly(
result,
workspace,
seedTableData,
);
}
}
filterResults(result, options) {
for (const workspaceId in result.activityReport) {
for (const workspaceId in result) {
const timeDifferenceInSeconds = Math.abs(
new Date().getTime() -
new Date(result.activityReport[workspaceId].maxUpdatedAt).getTime(),
new Date(result[workspaceId].maxUpdatedAt).getTime(),
);
const timeDifferenceInDays = Math.ceil(
timeDifferenceInSeconds / (1000 * 3600 * 24),
);
if (timeDifferenceInDays < options.sameAsSeedDays) {
delete result.sameAsSeedWorkspaces[workspaceId];
}
if (timeDifferenceInDays < options.days) {
delete result.activityReport[workspaceId];
if (
timeDifferenceInDays < options.days &&
(!result[workspaceId].sameAsSeed ||
timeDifferenceInDays < options.sameAsSeedDays)
) {
delete result[workspaceId];
} else {
result.activityReport[workspaceId].inactiveDays = timeDifferenceInDays;
result[workspaceId].inactiveDays = timeDifferenceInDays;
}
}
}
async delete(result) {
if (Object.keys(result.activityReport).length) {
console.log('Deleting inactive workspaces');
async delete(result, options) {
const workspaceCount = Object.keys(result).length;
if (workspaceCount) {
console.log(
`Deleting \x1b[36m${workspaceCount}\x1b[0m inactive since \x1b[36m${options.days} days\x1b[0m or same as seed since \x1b[36m${options.sameAsSeedDays} days\x1b[0m workspaces`,
);
}
for (const workspaceId in result.activityReport) {
let count = 1;
for (const workspaceId in result) {
process.stdout.write(`- deleting ${workspaceId} ...`);
await this.workspaceService.deleteWorkspace({
workspaceId,
});
console.log(' done!');
}
if (Object.keys(result.sameAsSeedWorkspaces).length) {
console.log('Deleting same as Seed workspaces');
}
for (const workspaceId in result.sameAsSeedWorkspaces) {
process.stdout.write(`- deleting ${workspaceId} ...`);
await this.workspaceService.deleteWorkspace({
workspaceId,
});
console.log(' done!');
console.log(
` done! ....... ${Math.floor((100 * count) / workspaceCount)}%`,
);
count += 1;
}
}
displayResults(result) {
const workspacesToDelete = new Set();
for (const workspaceId in result.activityReport) {
workspacesToDelete.add(workspaceId);
}
for (const workspaceId in result.sameAsSeedWorkspaces) {
workspacesToDelete.add(workspaceId);
}
console.log(`${workspacesToDelete.size} workspace(s) will be deleted:`);
displayResults(result, totalWorkspacesCount) {
console.log(result);
console.log(
`${
Object.keys(result).length
} out of ${totalWorkspacesCount} workspace(s) checked (${Math.floor(
(100 * Object.keys(result).length) / totalWorkspacesCount,
)}%) will be deleted`,
);
}
async run(
_passedParam: string[],
options: DataCleanInactiveOptions,
): Promise<void> {
const result: DataCleanResults = {
activityReport: {},
sameAsSeedWorkspaces: {},
};
await this.findInactiveWorkspaces(result, options);
const result: DataCleanResults = {};
const workspaces = await this.getWorkspaces(options);
const totalWorkspacesCount = workspaces.length;
console.log(totalWorkspacesCount, 'workspace(s) to analyse');
await this.findInactiveWorkspaces(workspaces, result);
this.filterResults(result, options);
this.displayResults(result);
this.displayResults(result, totalWorkspacesCount);
if (!options.dryRun) {
options = await this.inquiererService.ask('confirm', options);
if (!options.confirmation) {
@ -257,7 +338,7 @@ export class DataCleanInactiveCommand extends CommandRunner {
}
}
if (!options.dryRun) {
await this.delete(result);
await this.delete(result, options);
}
}
}

3
server/src/metadata/data-source/data-source.service.ts
View File

@ -127,7 +127,8 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy {
const schemaAlreadyExists = await queryRunner.hasSchema(schemaName);
if (!schemaAlreadyExists) {
throw new Error(`Schema ${schemaName} does not exist`);
await queryRunner.release();
return;
}
await queryRunner.dropSchema(schemaName, true, true);

4
server/src/metadata/field-metadata/services/field-metadata.service.ts
View File

@ -103,4 +103,8 @@ export class FieldMetadataService extends TypeOrmQueryService<FieldMetadata> {
return createdFieldMetadata;
}
public async deleteFieldsMetadata(workspaceId: string) {
await this.fieldMetadataRepository.delete({ workspaceId });
}
}

2
server/src/metadata/metadata.datasource.ts
View File

@ -15,7 +15,7 @@ export const typeORMMetadataModuleOptions: TypeOrmModuleOptions = {
schema: 'metadata',
entities: [__dirname + '/**/*.entity{.ts,.js}'],
synchronize: false,
migrationsRun: true,
migrationsRun: false,
migrationsTableName: '_typeorm_migrations',
migrations: [__dirname + '/migrations/*{.ts,.js}'],
};

2
server/src/metadata/object-metadata/services/object-metadata.service.ts
View File

@ -131,7 +131,7 @@ export class ObjectMetadataService extends TypeOrmQueryService<ObjectMetadata> {
);
}
public async deleteObjectsAndFieldsMetadata(workspaceId: string) {
public async deleteObjectsMetadata(workspaceId: string) {
await this.objectMetadataRepository.delete({ workspaceId });
}
}

2
server/src/metadata/tenant-initialisation/tenant-initialisation.module.ts
View File

@ -5,6 +5,7 @@ import { MigrationRunnerModule } from 'src/metadata/migration-runner/migration-r
import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module';
import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module';
import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module';
import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module';
import { TenantInitialisationService } from './tenant-initialisation.service';
@ -14,6 +15,7 @@ import { TenantInitialisationService } from './tenant-initialisation.service';
TenantMigrationModule,
MigrationRunnerModule,
ObjectMetadataModule,
FieldMetadataModule,
DataSourceMetadataModule,
],
exports: [TenantInitialisationService],

7
server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts
View File

@ -6,6 +6,7 @@ import { DataSourceService } from 'src/metadata/data-source/data-source.service'
import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service';
import { ObjectMetadataService } from 'src/metadata/object-metadata/services/object-metadata.service';
import { DataSourceMetadata } from 'src/metadata/data-source-metadata/data-source-metadata.entity';
import { FieldMetadataService } from 'src/metadata/field-metadata/services/field-metadata.service';
import { standardObjectsPrefillData } from './standard-objects-prefill-data/standard-objects-prefill-data';
@ -16,6 +17,7 @@ export class TenantInitialisationService {
private readonly tenantMigrationService: TenantMigrationService,
private readonly migrationRunnerService: MigrationRunnerService,
private readonly objectMetadataService: ObjectMetadataService,
private readonly fieldMetadataService: FieldMetadataService,
private readonly dataSourceMetadataService: DataSourceMetadataService,
) {}
@ -78,9 +80,8 @@ export class TenantInitialisationService {
public async delete(workspaceId: string): Promise<void> {
// Delete data from metadata tables
await this.objectMetadataService.deleteObjectsAndFieldsMetadata(
workspaceId,
);
await this.fieldMetadataService.deleteFieldsMetadata(workspaceId);
await this.objectMetadataService.deleteObjectsMetadata(workspaceId);
await this.tenantMigrationService.delete(workspaceId);
await this.dataSourceMetadataService.delete(workspaceId);
// Delete schema