diff --git a/server/package.json b/server/package.json index 6e214b705..f21b6bd88 100644 --- a/server/package.json +++ b/server/package.json @@ -35,7 +35,8 @@ "database:migrate": "yarn typeorm:migrate && yarn prisma:migrate", "database:generate": "yarn prisma:generate", "database:seed": "yarn prisma:seed", - "database:reset": "yarn database:truncate && yarn database:init" + "database:reset": "yarn database:truncate && yarn database:init", + "command": "node dist/src/command" }, "dependencies": { "@apollo/server": "^4.7.3", @@ -91,6 +92,7 @@ "lodash.snakecase": "^4.1.1", "lodash.upperfirst": "^4.3.1", "ms": "^2.1.3", + "nest-commander": "^3.12.0", "passport": "^0.6.0", "passport-google-oauth20": "^2.0.0", "passport-jwt": "^4.0.1", @@ -157,4 +159,4 @@ "schema": "src/database/schema.prisma", "seed": "ts-node src/database/seeds/index.ts" } -} \ No newline at end of file +} diff --git a/server/src/command.module.ts b/server/src/command.module.ts new file mode 100644 index 000000000..1bd641d2a --- /dev/null +++ b/server/src/command.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; + +import { AppModule } from './app.module'; + +import { MetadataCommandModule } from './metadata/commands/metadata-command.module'; + +@Module({ + imports: [AppModule, MetadataCommandModule], +}) +export class CommandModule {} diff --git a/server/src/command.ts b/server/src/command.ts new file mode 100644 index 000000000..981ac1e2d --- /dev/null +++ b/server/src/command.ts @@ -0,0 +1,9 @@ +import { CommandFactory } from 'nest-commander'; + +import { CommandModule } from './command.module'; + +async function bootstrap() { + // TODO: inject our own logger service to handle the output (Sentry, etc.) + await CommandFactory.run(CommandModule, ['warn', 'error']); +} +bootstrap(); diff --git a/server/src/metadata/commands/metadata-command.module.ts b/server/src/metadata/commands/metadata-command.module.ts new file mode 100644 index 000000000..423b236b0 --- /dev/null +++ b/server/src/metadata/commands/metadata-command.module.ts @@ -0,0 +1,24 @@ +import { Module } from '@nestjs/common'; + +import { TenantMigrationModule } from 'src/metadata/tenant-migration/tenant-migration.module'; +import { MigrationRunnerModule } from 'src/metadata/migration-runner/migration-runner.module'; +import { ObjectMetadataModule } from 'src/metadata/object-metadata/object-metadata.module'; +import { FieldMetadataModule } from 'src/metadata/field-metadata/field-metadata.module'; +import { TenantInitialisationModule } from 'src/metadata/tenant-initialisation/tenant-initialisation.module'; +import { DataSourceMetadataModule } from 'src/metadata/data-source-metadata/data-source-metadata.module'; + +import { SyncTenantMetadataCommand } from './sync-tenant-metadata.command'; +import { RunTenantMigrationsCommand } from './run-tenant-migrations.command'; + +@Module({ + imports: [ + TenantMigrationModule, + MigrationRunnerModule, + ObjectMetadataModule, + FieldMetadataModule, + DataSourceMetadataModule, + TenantInitialisationModule, + ], + providers: [RunTenantMigrationsCommand, SyncTenantMetadataCommand], +}) +export class MetadataCommandModule {} diff --git a/server/src/metadata/commands/run-tenant-migrations.command.ts b/server/src/metadata/commands/run-tenant-migrations.command.ts new file mode 100644 index 000000000..557a23574 --- /dev/null +++ b/server/src/metadata/commands/run-tenant-migrations.command.ts @@ -0,0 +1,45 @@ +import { Command, CommandRunner, Option } from 'nest-commander'; + +import { TenantMigrationService } from 'src/metadata/tenant-migration/tenant-migration.service'; +import { MigrationRunnerService } from 'src/metadata/migration-runner/migration-runner.service'; + +// TODO: implement dry-run +interface RunTenantMigrationsOptions { + workspaceId: string; +} + +@Command({ + name: 'tenant:migrate', + description: 'Run tenant migrations', +}) +export class RunTenantMigrationsCommand extends CommandRunner { + constructor( + private readonly tenantMigrationService: TenantMigrationService, + private readonly migrationRunnerService: MigrationRunnerService, + ) { + super(); + } + + async run( + _passedParam: string[], + options: RunTenantMigrationsOptions, + ): Promise { + // TODO: run in a dedicated job + run queries in a transaction. + await this.tenantMigrationService.insertStandardMigrations( + options.workspaceId, + ); + await this.migrationRunnerService.executeMigrationFromPendingMigrations( + options.workspaceId, + ); + } + + // TODO: workspaceId should be optional and we should run migrations for all workspaces + @Option({ + flags: '-w, --workspace-id [workspace_id]', + description: 'workspace id', + required: true, + }) + parseWorkspaceId(value: string): string { + return value; + } +} diff --git a/server/src/metadata/commands/sync-tenant-metadata.command.ts b/server/src/metadata/commands/sync-tenant-metadata.command.ts new file mode 100644 index 000000000..6d3470dbe --- /dev/null +++ b/server/src/metadata/commands/sync-tenant-metadata.command.ts @@ -0,0 +1,57 @@ +import { Command, CommandRunner, Option } from 'nest-commander'; + +import { ObjectMetadataService } from 'src/metadata/object-metadata/services/object-metadata.service'; +import { FieldMetadataService } from 'src/metadata/field-metadata/services/field-metadata.service'; +import { TenantInitialisationService } from 'src/metadata/tenant-initialisation/tenant-initialisation.service'; +import { DataSourceMetadataService } from 'src/metadata/data-source-metadata/data-source-metadata.service'; + +// TODO: implement dry-run +interface RunTenantMigrationsOptions { + workspaceId: string; +} + +@Command({ + name: 'tenant:sync-metadata', + description: 'Sync metadata', +}) +export class SyncTenantMetadataCommand extends CommandRunner { + constructor( + private readonly objectMetadataService: ObjectMetadataService, + private readonly fieldMetadataService: FieldMetadataService, + private readonly dataSourceMetadataService: DataSourceMetadataService, + private readonly tenantInitialisationService: TenantInitialisationService, + ) { + super(); + } + + async run( + _passedParam: string[], + options: RunTenantMigrationsOptions, + ): Promise { + // TODO: run in a dedicated job + run queries in a transaction. + const dataSourceMetadata = + await this.dataSourceMetadataService.getLastDataSourceMetadataFromWorkspaceIdOrFail( + options.workspaceId, + ); + + // TODO: This solution could be improved, using a diff for example, we should not have to delete all metadata and recreate them. + await this.objectMetadataService.deleteMany({ + workspaceId: { eq: options.workspaceId }, + }); + + // TODO: this should not be the responsibility of tenantInitialisationService. + await this.tenantInitialisationService.createObjectsAndFieldsMetadata( + dataSourceMetadata.id, + options.workspaceId, + ); + } + + @Option({ + flags: '-w, --workspace-id [workspace_id]', + description: 'workspace id', + required: true, + }) + parseWorkspaceId(value: string): string { + return value; + } +} diff --git a/server/src/metadata/data-source/data-source.service.ts b/server/src/metadata/data-source/data-source.service.ts index 3000f138b..deb7ee8b6 100644 --- a/server/src/metadata/data-source/data-source.service.ts +++ b/server/src/metadata/data-source/data-source.service.ts @@ -132,5 +132,10 @@ export class DataSourceService implements OnModuleInit, OnModuleDestroy { async onModuleDestroy() { // Destroy main data source "default" schema await this.mainDataSource.destroy(); + + // Destroy all workspace data sources + for (const [, dataSource] of this.dataSources) { + await dataSource.destroy(); + } } } diff --git a/server/src/metadata/migration-runner/migration-runner.service.ts b/server/src/metadata/migration-runner/migration-runner.service.ts index 35d736e3b..c85130b33 100644 --- a/server/src/metadata/migration-runner/migration-runner.service.ts +++ b/server/src/metadata/migration-runner/migration-runner.service.ts @@ -35,6 +35,10 @@ export class MigrationRunnerService { const pendingMigrations = await this.tenantMigrationService.getPendingMigrations(workspaceId); + if (pendingMigrations.length === 0) { + return []; + } + const flattenedPendingMigrations: TenantMigrationTableAction[] = pendingMigrations.reduce((acc, pendingMigration) => { return [...acc, ...pendingMigration.migrations]; diff --git a/server/src/metadata/tenant-initialisation/standard-objects/companies.metadata.json b/server/src/metadata/tenant-initialisation/standard-objects/companies/companies.metadata.json similarity index 100% rename from server/src/metadata/tenant-initialisation/standard-objects/companies.metadata.json rename to server/src/metadata/tenant-initialisation/standard-objects/companies/companies.metadata.json diff --git a/server/src/metadata/tenant-initialisation/standard-objects/companies.seeds.json b/server/src/metadata/tenant-initialisation/standard-objects/companies/companies.seeds.json similarity index 100% rename from server/src/metadata/tenant-initialisation/standard-objects/companies.seeds.json rename to server/src/metadata/tenant-initialisation/standard-objects/companies/companies.seeds.json diff --git a/server/src/metadata/tenant-initialisation/standard-objects/people/people.metadata.json b/server/src/metadata/tenant-initialisation/standard-objects/people/people.metadata.json new file mode 100644 index 000000000..9d4000349 --- /dev/null +++ b/server/src/metadata/tenant-initialisation/standard-objects/people/people.metadata.json @@ -0,0 +1,113 @@ +{ + "nameSingular": "personV2", + "namePlural": "peopleV2", + "labelSingular": "Person", + "labelPlural": "People", + "targetTableName": "person", + "description": "A person", + "icon": "people", + "fields": [ + { + "type": "text", + "name": "firstName", + "label": "First Name", + "targetColumnMap": { + "value": "firstName" + }, + "description": "First Name of the person", + "icon": null, + "isNullable": true + }, + { + "type": "text", + "name": "lastName", + "label": "Last Name", + "targetColumnMap": { + "value": "lastName" + }, + "description": "Last Name of the person", + "icon": null, + "isNullable": true + }, + { + "type": "text", + "name": "email", + "label": "Email", + "targetColumnMap": { + "value": "email" + }, + "description": "Email of the person", + "icon": null, + "isNullable": true + }, + { + "type": "phone", + "name": "phone", + "label": "Phone", + "targetColumnMap": { + "value": "phone" + }, + "description": "phone of the company", + "icon": null, + "isNullable": true + }, + { + "type": "text", + "name": "city", + "label": "City", + "targetColumnMap": { + "value": "city" + }, + "description": "City of the person", + "icon": null, + "isNullable": true + }, + { + "type": "text", + "name": "jobTitle", + "label": "Job Title", + "targetColumnMap": { + "value": "jobTitle" + }, + "description": "Job title of the person", + "icon": null, + "isNullable": true + }, + { + "type": "url", + "name": "linkedinUrl", + "label": "Linkedin URL", + "targetColumnMap": { + "text": "Linkedin URL", + "link": "linkedinUrl" + }, + "description": "Linkedin URL of the person", + "icon": "url", + "isNullable": true + }, + { + "type": "url", + "name": "xUrl", + "label": "X URL", + "targetColumnMap": { + "text": "X URL", + "link": "xUrl" + }, + "description": "X URL of the person", + "icon": "url", + "isNullable": true + }, + { + "type": "url", + "name": "avatarUrl", + "label": "Avatar URL", + "targetColumnMap": { + "text": "Avatar URL", + "link": "avatarUrl" + }, + "description": "Avatar URL of the person", + "icon": "url", + "isNullable": true + } + ] +} \ No newline at end of file diff --git a/server/src/metadata/tenant-initialisation/standard-objects/standard-object-metadata.ts b/server/src/metadata/tenant-initialisation/standard-objects/standard-object-metadata.ts index 04166f63f..2c27b0560 100644 --- a/server/src/metadata/tenant-initialisation/standard-objects/standard-object-metadata.ts +++ b/server/src/metadata/tenant-initialisation/standard-objects/standard-object-metadata.ts @@ -1,5 +1,7 @@ -import companyObject from './companies.metadata.json'; +import companyObject from './companies/companies.metadata.json'; +import personObject from './people/people.metadata.json'; export const standardObjectsMetadata = { companyV2: companyObject, + personV2: personObject, }; diff --git a/server/src/metadata/tenant-initialisation/standard-objects/standard-object-seeds.ts b/server/src/metadata/tenant-initialisation/standard-objects/standard-object-seeds.ts index 621587c1c..6b8d6ab0b 100644 --- a/server/src/metadata/tenant-initialisation/standard-objects/standard-object-seeds.ts +++ b/server/src/metadata/tenant-initialisation/standard-objects/standard-object-seeds.ts @@ -1,4 +1,4 @@ -import companySeeds from './companies.seeds.json'; +import companySeeds from './companies/companies.seeds.json'; export const standardObjectsSeeds = { companyV2: companySeeds, diff --git a/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts b/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts index 37943de6a..53393b427 100644 --- a/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts +++ b/server/src/metadata/tenant-initialisation/tenant-initialisation.service.ts @@ -67,7 +67,7 @@ export class TenantInitialisationService { * @param dataSourceMetadataId * @param workspaceId */ - private async createObjectsAndFieldsMetadata( + public async createObjectsAndFieldsMetadata( dataSourceMetadataId: string, workspaceId: string, ) { diff --git a/server/src/metadata/tenant-migration/migrations/1697618010-addPeopleTable.ts b/server/src/metadata/tenant-migration/migrations/1697618010-addPeopleTable.ts new file mode 100644 index 000000000..78745a780 --- /dev/null +++ b/server/src/metadata/tenant-migration/migrations/1697618010-addPeopleTable.ts @@ -0,0 +1,59 @@ +import { TenantMigrationTableAction } from 'src/metadata/tenant-migration/tenant-migration.entity'; + +export const addPeopleTable: TenantMigrationTableAction[] = [ + { + name: 'people', + action: 'create', + }, + { + name: 'people', + action: 'alter', + columns: [ + { + name: 'firstName', + type: 'varchar', + action: 'create', + }, + { + name: 'lastName', + type: 'varchar', + action: 'create', + }, + { + name: 'email', + type: 'varchar', + action: 'create', + }, + { + name: 'phone', + type: 'varchar', + action: 'create', + }, + { + name: 'city', + type: 'varchar', + action: 'create', + }, + { + name: 'jobTitle', + type: 'varchar', + action: 'create', + }, + { + name: 'linkedinUrl', + type: 'varchar', + action: 'create', + }, + { + name: 'xUrl', + type: 'varchar', + action: 'create', + }, + { + name: 'avatarUrl', + type: 'varchar', + action: 'create', + }, + ], + }, +]; diff --git a/server/src/metadata/tenant-migration/standard-migrations.ts b/server/src/metadata/tenant-migration/standard-migrations.ts index 47b9dca59..c89a58924 100644 --- a/server/src/metadata/tenant-migration/standard-migrations.ts +++ b/server/src/metadata/tenant-migration/standard-migrations.ts @@ -1,6 +1,8 @@ import { addCompanyTable } from './migrations/1697618009-addCompanyTable'; +import { addPeopleTable } from './migrations/1697618010-addPeopleTable'; // TODO: read the folder and return all migrations export const standardMigrations = { '1697618009-addCompanyTable': addCompanyTable, + '1697618010-addPeopleTable': addPeopleTable, }; diff --git a/server/src/metadata/tenant-migration/tenant-migration.service.ts b/server/src/metadata/tenant-migration/tenant-migration.service.ts index 36074d397..37ff0a194 100644 --- a/server/src/metadata/tenant-migration/tenant-migration.service.ts +++ b/server/src/metadata/tenant-migration/tenant-migration.service.ts @@ -21,7 +21,7 @@ export class TenantMigrationService { * * @param workspaceId */ - public async insertStandardMigrations(workspaceId: string) { + public async insertStandardMigrations(workspaceId: string): Promise { // TODO: we actually don't need to fetch all of them, to improve later so it scales well. const insertedStandardMigrations = await this.tenantMigrationRepository.find({ @@ -34,20 +34,21 @@ export class TenantMigrationService { return acc; }, {}); - const standardMigrationsList = standardMigrations; - const standardMigrationsListThatNeedToBeInserted = Object.entries( - standardMigrationsList, + standardMigrations, ) .filter(([name]) => !insertedStandardMigrationsMapByName[name]) .map(([name, migrations]) => ({ name, migrations })); - await this.tenantMigrationRepository.save( + const standardMigrationsThatNeedToBeInserted = standardMigrationsListThatNeedToBeInserted.map((migration) => ({ ...migration, workspaceId, isCustom: false, - })), + })); + + await this.tenantMigrationRepository.save( + standardMigrationsThatNeedToBeInserted, ); } @@ -60,7 +61,7 @@ export class TenantMigrationService { public async getPendingMigrations( workspaceId: string, ): Promise { - return this.tenantMigrationRepository.find({ + return await this.tenantMigrationRepository.find({ order: { createdAt: 'ASC' }, where: { appliedAt: IsNull(), diff --git a/server/yarn.lock b/server/yarn.lock index 6f509e9db..e169b6aac 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -1329,6 +1329,20 @@ resolved "https://registry.npmjs.org/@eslint/js/-/js-8.42.0.tgz" integrity sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw== +"@fig/complete-commander@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@fig/complete-commander/-/complete-commander-2.0.1.tgz#6dd84f8812389107529aaedebd1bb67ac8bc16c6" + integrity sha512-AbGETely7iwD4F7XHe4g7pW6icWYYqJMdQog8CdEi9syU/av5L0O24BvCfgEeGO6TRPMpC+rFL7ZDJsqRtckOA== + dependencies: + prettier "^2.3.2" + +"@golevelup/nestjs-discovery@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@golevelup/nestjs-discovery/-/nestjs-discovery-4.0.0.tgz#3428f0b620b51e4d425bc9e41cc8f2f338472dc1" + integrity sha512-iyZLYip9rhVMR0C93vo860xmboRrD5g5F5iEOfpeblGvYSz8ymQrL9RAST7x/Fp3n+TAXSeOLzDIASt+rak68g== + dependencies: + lodash "^4.17.21" + "@graphql-tools/executor@^1.0.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@graphql-tools/executor/-/executor-1.2.0.tgz#6c45f4add765769d9820c4c4405b76957ba39c79" @@ -4507,6 +4521,11 @@ combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" +commander@11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-11.0.0.tgz#43e19c25dbedc8256203538e8d7e9346877a6f67" + integrity sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ== + commander@4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz" @@ -4612,6 +4631,16 @@ cors@2.8.5, cors@^2.8.5: object-assign "^4" vary "^1" +cosmiconfig@8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.2.0.tgz#f7d17c56a590856cd1e7cee98734dca272b0d8fd" + integrity sha512-3rTMnFJA1tCOPwRxtgF4wd7Ab2qvDbL8jX+3smjIbS4HlZBagTlpERbdN7iAbWlrfxE3M8c27kTwTawQ7st+OQ== + dependencies: + import-fresh "^3.2.1" + js-yaml "^4.1.0" + parse-json "^5.0.0" + path-type "^4.0.0" + cosmiconfig@^7.0.1: version "7.1.0" resolved "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz" @@ -7389,6 +7418,17 @@ neo-async@^2.6.2: resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== +nest-commander@^3.12.0: + version "3.12.0" + resolved "https://registry.yarnpkg.com/nest-commander/-/nest-commander-3.12.0.tgz#9d1d7df7c9fa129d899c1e85c49eeb749bd11376" + integrity sha512-6ncAT13l7lH9Hya3GKKOIG+ltRD7b4idTlbuNXaCsm2IJIuuVxnx35UxiogJPz+GarE437H3I+GJXzehBnDQqg== + dependencies: + "@fig/complete-commander" "^2.0.1" + "@golevelup/nestjs-discovery" "4.0.0" + commander "11.0.0" + cosmiconfig "8.2.0" + inquirer "8.2.5" + new-github-issue-url@0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/new-github-issue-url/-/new-github-issue-url-0.2.1.tgz"