Api keys and webhook migration to core (#13011)
TODO: check Zapier trigger records work as expected --------- Co-authored-by: Weiko <corentin@twenty.com>
This commit is contained in:
nitin
@ -1,10 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { CronRegisterAllCommand } from 'src/database/commands/cron-register-all.command';
|
||||
import { DataSeedWorkspaceCommand } from 'src/database/commands/data-seed-dev-workspace.command';
|
||||
import { ConfirmationQuestion } from 'src/database/commands/questions/confirmation.question';
|
||||
import { UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/upgrade-version-command.module';
|
||||
import { TypeORMModule } from 'src/database/typeorm/typeorm.module';
|
||||
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
|
||||
import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module';
|
||||
import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module';
|
||||
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
|
||||
@ -15,6 +15,8 @@ import { CalendarEventImportManagerModule } from 'src/modules/calendar/calendar-
|
||||
import { MessagingImportManagerModule } from 'src/modules/messaging/message-import-manager/messaging-import-manager.module';
|
||||
import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/automated-trigger/automated-trigger.module';
|
||||
|
||||
import { DataSeedWorkspaceCommand } from './data-seed-dev-workspace.command';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
UpgradeVersionCommandModule,
|
||||
@ -24,7 +26,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
|
||||
CalendarEventImportManagerModule,
|
||||
AutomatedTriggerModule,
|
||||
|
||||
// Only needed for the data seed command
|
||||
// Data seeding dependencies
|
||||
TypeORMModule,
|
||||
FieldMetadataModule,
|
||||
ObjectMetadataModule,
|
||||
@ -32,6 +34,7 @@ import { AutomatedTriggerModule } from 'src/modules/workflow/workflow-trigger/au
|
||||
WorkspaceManagerModule,
|
||||
DataSourceModule,
|
||||
WorkspaceCacheStorageModule,
|
||||
ApiKeyModule,
|
||||
],
|
||||
providers: [
|
||||
DataSeedWorkspaceCommand,
|
||||
|
||||
@ -0,0 +1,212 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Command } from 'nest-commander';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import {
|
||||
ActiveOrSuspendedWorkspacesMigrationCommandRunner,
|
||||
RunOnWorkspaceArgs,
|
||||
} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
|
||||
import { ApiKey } from 'src/engine/core-modules/api-key/api-key.entity';
|
||||
import { ApiKeyService } from 'src/engine/core-modules/api-key/api-key.service';
|
||||
import { Webhook } from 'src/engine/core-modules/webhook/webhook.entity';
|
||||
import { WebhookService } from 'src/engine/core-modules/webhook/webhook.service';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
import { ApiKeyWorkspaceEntity } from 'src/modules/api-key/standard-objects/api-key.workspace-entity';
|
||||
import { WebhookWorkspaceEntity } from 'src/modules/webhook/standard-objects/webhook.workspace-entity';
|
||||
|
||||
@Command({
|
||||
name: 'upgrade:1-3:migrate-api-keys-webhooks-to-core',
|
||||
description:
|
||||
'Migrate API keys and webhooks from workspace schemas to core schema',
|
||||
})
|
||||
export class MigrateApiKeysWebhooksToCoreCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
|
||||
constructor(
|
||||
@InjectRepository(Workspace, 'core')
|
||||
protected readonly workspaceRepository: Repository<Workspace>,
|
||||
@InjectRepository(ApiKey, 'core')
|
||||
private readonly coreApiKeyRepository: Repository<ApiKey>,
|
||||
@InjectRepository(Webhook, 'core')
|
||||
private readonly coreWebhookRepository: Repository<Webhook>,
|
||||
private readonly apiKeyService: ApiKeyService,
|
||||
private readonly webhookService: WebhookService,
|
||||
protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
|
||||
) {
|
||||
super(workspaceRepository, twentyORMGlobalManager);
|
||||
}
|
||||
|
||||
override async runOnWorkspace({
|
||||
index,
|
||||
total,
|
||||
workspaceId,
|
||||
options,
|
||||
}: RunOnWorkspaceArgs): Promise<void> {
|
||||
this.logger.log(
|
||||
`Migrating API keys and webhooks for workspace ${workspaceId} ${index + 1}/${total}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.migrateApiKeys(workspaceId, options.dryRun);
|
||||
|
||||
await this.migrateWebhooks(workspaceId, options.dryRun);
|
||||
|
||||
this.logger.log(
|
||||
`Successfully migrated API keys and webhooks for workspace ${workspaceId}`,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to migrate API keys and webhooks for workspace ${workspaceId}: ${error.message}`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async migrateApiKeys(
|
||||
workspaceId: string,
|
||||
dryRun?: boolean,
|
||||
): Promise<void> {
|
||||
const workspaceApiKeyRepository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace<ApiKeyWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'apiKey',
|
||||
{ shouldBypassPermissionChecks: true },
|
||||
);
|
||||
|
||||
const workspaceApiKeys = await workspaceApiKeyRepository.find({
|
||||
withDeleted: true,
|
||||
});
|
||||
|
||||
if (workspaceApiKeys.length === 0) {
|
||||
this.logger.log(`No API keys to migrate for workspace ${workspaceId}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`${dryRun ? 'DRY RUN: ' : ''}Found ${workspaceApiKeys.length} API keys to migrate for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
if (dryRun) {
|
||||
workspaceApiKeys.forEach((apiKey) => {
|
||||
const deletedStatus = apiKey.deletedAt ? ' (DELETED)' : '';
|
||||
|
||||
this.logger.log(
|
||||
`DRY RUN: Would migrate API key ${apiKey.id} (${apiKey.name})${deletedStatus} from workspace ${workspaceId}`,
|
||||
);
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const existingCoreApiKeys = await this.coreApiKeyRepository.find({
|
||||
where: { workspaceId },
|
||||
select: ['id'],
|
||||
withDeleted: true,
|
||||
});
|
||||
const existingApiKeyIds = new Set(existingCoreApiKeys.map((ak) => ak.id));
|
||||
|
||||
for (const workspaceApiKey of workspaceApiKeys) {
|
||||
if (existingApiKeyIds.has(workspaceApiKey.id)) {
|
||||
this.logger.warn(
|
||||
`API key ${workspaceApiKey.id} already exists in core schema for workspace ${workspaceId}, skipping`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.apiKeyService.create({
|
||||
id: workspaceApiKey.id,
|
||||
name: workspaceApiKey.name,
|
||||
expiresAt: workspaceApiKey.expiresAt,
|
||||
revokedAt: workspaceApiKey.revokedAt
|
||||
? new Date(workspaceApiKey.revokedAt)
|
||||
: workspaceApiKey.deletedAt
|
||||
? new Date(workspaceApiKey.deletedAt)
|
||||
: undefined,
|
||||
workspaceId,
|
||||
createdAt: new Date(workspaceApiKey.createdAt),
|
||||
updatedAt: new Date(workspaceApiKey.updatedAt),
|
||||
});
|
||||
|
||||
const deletedStatus = workspaceApiKey.deletedAt ? ' (DELETED)' : '';
|
||||
|
||||
this.logger.log(
|
||||
`Migrated API key ${workspaceApiKey.id} (${workspaceApiKey.name})${deletedStatus} to core schema`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async migrateWebhooks(
|
||||
workspaceId: string,
|
||||
dryRun?: boolean,
|
||||
): Promise<void> {
|
||||
const workspaceWebhookRepository =
|
||||
await this.twentyORMGlobalManager.getRepositoryForWorkspace<WebhookWorkspaceEntity>(
|
||||
workspaceId,
|
||||
'webhook',
|
||||
{ shouldBypassPermissionChecks: true },
|
||||
);
|
||||
|
||||
const workspaceWebhooks = await workspaceWebhookRepository.find({
|
||||
withDeleted: true,
|
||||
});
|
||||
|
||||
if (workspaceWebhooks.length === 0) {
|
||||
this.logger.log(`No webhooks to migrate for workspace ${workspaceId}`);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`${dryRun ? 'DRY RUN: ' : ''}Found ${workspaceWebhooks.length} webhooks to migrate for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
if (dryRun) {
|
||||
workspaceWebhooks.forEach((webhook) => {
|
||||
const deletedStatus = webhook.deletedAt ? ' (DELETED)' : '';
|
||||
|
||||
this.logger.log(
|
||||
`DRY RUN: Would migrate webhook ${webhook.id} (${webhook.targetUrl})${deletedStatus} from workspace ${workspaceId}`,
|
||||
);
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const existingCoreWebhooks = await this.coreWebhookRepository.find({
|
||||
where: { workspaceId },
|
||||
select: ['id'],
|
||||
withDeleted: true,
|
||||
});
|
||||
const existingWebhookIds = new Set(existingCoreWebhooks.map((wh) => wh.id));
|
||||
|
||||
for (const workspaceWebhook of workspaceWebhooks) {
|
||||
if (existingWebhookIds.has(workspaceWebhook.id)) {
|
||||
this.logger.warn(
|
||||
`Webhook ${workspaceWebhook.id} already exists in core schema for workspace ${workspaceId}, skipping`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.webhookService.create({
|
||||
id: workspaceWebhook.id,
|
||||
targetUrl: workspaceWebhook.targetUrl,
|
||||
operations: workspaceWebhook.operations,
|
||||
description: workspaceWebhook.description,
|
||||
secret: workspaceWebhook.secret,
|
||||
workspaceId,
|
||||
createdAt: new Date(workspaceWebhook.createdAt),
|
||||
updatedAt: new Date(workspaceWebhook.updatedAt),
|
||||
deletedAt: workspaceWebhook.deletedAt
|
||||
? new Date(workspaceWebhook.deletedAt)
|
||||
: undefined,
|
||||
});
|
||||
|
||||
const deletedStatus = workspaceWebhook.deletedAt ? ' (DELETED)' : '';
|
||||
|
||||
this.logger.log(
|
||||
`Migrated webhook ${workspaceWebhook.id} (${workspaceWebhook.targetUrl})${deletedStatus} to core schema`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { MigrateApiKeysWebhooksToCoreCommand } from 'src/database/commands/upgrade-version-command/1-3/1-3-migrate-api-keys-webhooks-to-core.command';
|
||||
import { ApiKey } from 'src/engine/core-modules/api-key/api-key.entity';
|
||||
import { ApiKeyModule } from 'src/engine/core-modules/api-key/api-key.module';
|
||||
import { Webhook } from 'src/engine/core-modules/webhook/webhook.entity';
|
||||
import { WebhookModule } from 'src/engine/core-modules/webhook/webhook.module';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([Workspace, ApiKey, Webhook], 'core'),
|
||||
WorkspaceDataSourceModule,
|
||||
ApiKeyModule,
|
||||
WebhookModule,
|
||||
],
|
||||
providers: [MigrateApiKeysWebhooksToCoreCommand],
|
||||
exports: [MigrateApiKeysWebhooksToCoreCommand],
|
||||
})
|
||||
export class V1_3_UpgradeVersionCommandModule {}
|
||||
@ -5,6 +5,7 @@ import { V0_54_UpgradeVersionCommandModule } from 'src/database/commands/upgrade
|
||||
import { V0_55_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/0-55/0-55-upgrade-version-command.module';
|
||||
import { V1_1_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-1/1-1-upgrade-version-command.module';
|
||||
import { V1_2_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-2/1-2-upgrade-version-command.module';
|
||||
import { V1_3_UpgradeVersionCommandModule } from 'src/database/commands/upgrade-version-command/1-3/1-3-upgrade-version-command.module';
|
||||
import {
|
||||
DatabaseMigrationService,
|
||||
UpgradeCommand,
|
||||
@ -19,6 +20,7 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp
|
||||
V0_55_UpgradeVersionCommandModule,
|
||||
V1_1_UpgradeVersionCommandModule,
|
||||
V1_2_UpgradeVersionCommandModule,
|
||||
V1_3_UpgradeVersionCommandModule,
|
||||
WorkspaceSyncMetadataModule,
|
||||
],
|
||||
providers: [DatabaseMigrationService, UpgradeCommand],
|
||||
|
||||
@ -24,6 +24,7 @@ import { DeduplicateIndexedFieldsCommand } from 'src/database/commands/upgrade-v
|
||||
import { FixSchemaArrayTypeCommand } from 'src/database/commands/upgrade-version-command/1-1/1-1-fix-schema-array-type.command';
|
||||
import { FixUpdateStandardFieldsIsLabelSyncedWithName } from 'src/database/commands/upgrade-version-command/1-1/1-1-fix-update-standard-field-is-label-synced-with-name.command';
|
||||
import { AddEnqueuedStatusToWorkflowRunCommand } from 'src/database/commands/upgrade-version-command/1-2/1-2-add-enqueued-status-to-workflow-run.command';
|
||||
import { MigrateApiKeysWebhooksToCoreCommand } from 'src/database/commands/upgrade-version-command/1-3/1-3-migrate-api-keys-webhooks-to-core.command';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
|
||||
@ -147,6 +148,9 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
// 1.2 Commands
|
||||
protected readonly migrateWorkflowRunStatesCommand: MigrateWorkflowRunStatesCommand,
|
||||
protected readonly addEnqueuedStatusToWorkflowRunCommand: AddEnqueuedStatusToWorkflowRunCommand,
|
||||
|
||||
// 1.3 Commands
|
||||
protected readonly migrateApiKeysWebhooksToCoreCommand: MigrateApiKeysWebhooksToCoreCommand,
|
||||
) {
|
||||
super(
|
||||
workspaceRepository,
|
||||
@ -200,6 +204,11 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
afterSyncMetadata: [this.migrateWorkflowRunStatesCommand],
|
||||
};
|
||||
|
||||
const commands_130: VersionCommands = {
|
||||
beforeSyncMetadata: [this.migrateApiKeysWebhooksToCoreCommand],
|
||||
afterSyncMetadata: [],
|
||||
};
|
||||
|
||||
this.allCommands = {
|
||||
'0.53.0': commands_053,
|
||||
'0.54.0': commands_054,
|
||||
@ -208,6 +217,7 @@ export class UpgradeCommand extends UpgradeCommandRunner {
|
||||
'1.0.0': commands_100,
|
||||
'1.1.0': commands_110,
|
||||
'1.2.0': commands_120,
|
||||
'1.3.0': commands_130,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,41 @@
|
||||
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||
|
||||
export class AddApiKeysAndWebhookToCore1751690946522
|
||||
implements MigrationInterface
|
||||
{
|
||||
name = 'AddApiKeysAndWebhookToCore1751690946522';
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE "core"."apiKey" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "name" character varying NOT NULL, "expiresAt" TIMESTAMP WITH TIME ZONE NOT NULL, "revokedAt" TIMESTAMP WITH TIME ZONE, "workspaceId" uuid NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "PK_2ae3a5e8e04fb402b2dc8d6ce4b" PRIMARY KEY ("id"))`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE INDEX "IDX_API_KEY_WORKSPACE_ID" ON "core"."apiKey" ("workspaceId") `,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE TABLE "core"."webhook" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "targetUrl" character varying NOT NULL, "operations" text array NOT NULL DEFAULT '{*.*}', "description" character varying, "secret" character varying NOT NULL, "workspaceId" uuid NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, CONSTRAINT "PK_e6765510c2d078db49632b59020" PRIMARY KEY ("id"))`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`CREATE INDEX "IDX_WEBHOOK_WORKSPACE_ID" ON "core"."webhook" ("workspaceId") `,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."apiKey" ADD CONSTRAINT "FK_c8b3efa54a29aa873043e72fb1d" FOREIGN KEY ("workspaceId") REFERENCES "core"."workspace"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."webhook" ADD CONSTRAINT "FK_597ab5e7de76f1836b8fd80d6b9" FOREIGN KEY ("workspaceId") REFERENCES "core"."workspace"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."webhook" DROP CONSTRAINT "FK_597ab5e7de76f1836b8fd80d6b9"`,
|
||||
);
|
||||
await queryRunner.query(
|
||||
`ALTER TABLE "core"."apiKey" DROP CONSTRAINT "FK_c8b3efa54a29aa873043e72fb1d"`,
|
||||
);
|
||||
await queryRunner.query(`DROP INDEX "core"."IDX_WEBHOOK_WORKSPACE_ID"`);
|
||||
await queryRunner.query(`DROP TABLE "core"."webhook"`);
|
||||
await queryRunner.query(`DROP INDEX "core"."IDX_API_KEY_WORKSPACE_ID"`);
|
||||
await queryRunner.query(`DROP TABLE "core"."apiKey"`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user