Add backfill position job by workspace (#5725)

- Removing existing listener that was backfilling created records without position - Switch to a job that backfill all objects within workspace - Adapting `FIND_BY_POSITION` so it can fetch objects without position. Currently we needed to input a number
2024-06-04 14:10:58 +02:00
parent b1f12d7257
commit 25f4e44aec
9 changed files with 307 additions and 104 deletions
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/tests/record-position-backfill-service.spec.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/tests/record-position-backfill-service.spec.ts
@ -0,0 +1,139 @@
+import { TestingModule, Test } from '@nestjs/testing';
+
+import { RecordPositionQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/record-position-query.factory';
+import { RecordPositionFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/record-position.factory';
+import { RecordPositionBackfillService } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-service';
+import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
+import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
+
+describe('RecordPositionBackfillService', () => {
+  let recordPositionQueryFactory;
+  let recordPositionFactory;
+  let objectMetadataService;
+  let workspaceDataSourceService;
+
+  let service: RecordPositionBackfillService;
+
+  beforeEach(async () => {
+    recordPositionQueryFactory = {
+      create: jest.fn().mockReturnValue(['query', []]),
+    };
+
+    recordPositionFactory = {
+      create: jest.fn().mockResolvedValue([
+        {
+          position: 1,
+        },
+      ]),
+    };
+
+    objectMetadataService = {
+      findManyWithinWorkspace: jest.fn().mockReturnValue([]),
+    };
+
+    workspaceDataSourceService = {
+      getSchemaName: jest.fn().mockReturnValue('schemaName'),
+      executeRawQuery: jest.fn().mockResolvedValue([]),
+    };
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        RecordPositionBackfillService,
+        {
+          provide: RecordPositionQueryFactory,
+          useValue: recordPositionQueryFactory,
+        },
+        {
+          provide: RecordPositionFactory,
+          useValue: recordPositionFactory,
+        },
+        {
+          provide: WorkspaceDataSourceService,
+          useValue: workspaceDataSourceService,
+        },
+        {
+          provide: ObjectMetadataService,
+          useValue: objectMetadataService,
+        },
+      ],
+    }).compile();
+
+    service = module.get<RecordPositionBackfillService>(
+      RecordPositionBackfillService,
+    );
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+
+  it('when no object metadata found, should do nothing', async () => {
+    await service.backfill('workspaceId', false);
+    expect(workspaceDataSourceService.executeRawQuery).not.toHaveBeenCalled();
+  });
+
+  it('when objectMetadata without position, should do nothing', async () => {
+    objectMetadataService.findManyWithinWorkspace.mockReturnValue([
+      {
+        id: '1',
+        nameSingular: 'name',
+        fields: [],
+      },
+    ]);
+    await service.backfill('workspaceId', false);
+    expect(workspaceDataSourceService.executeRawQuery).not.toHaveBeenCalled();
+  });
+
+  it('when objectMetadata but all record with position, should create and run query once', async () => {
+    objectMetadataService.findManyWithinWorkspace.mockReturnValue([
+      {
+        id: '1',
+        nameSingular: 'company',
+        fields: [],
+      },
+    ]);
+    await service.backfill('workspaceId', false);
+    expect(workspaceDataSourceService.executeRawQuery).toHaveBeenCalledTimes(1);
+  });
+
+  it('when record without position, should create and run query twice', async () => {
+    objectMetadataService.findManyWithinWorkspace.mockReturnValue([
+      {
+        id: '1',
+        nameSingular: 'company',
+        fields: [],
+      },
+    ]);
+    workspaceDataSourceService.executeRawQuery.mockResolvedValueOnce([
+      {
+        id: '1',
+      },
+    ]);
+    await service.backfill('workspaceId', false);
+    expect(workspaceDataSourceService.executeRawQuery).toHaveBeenCalledTimes(2);
+    expect(recordPositionFactory.create).toHaveBeenCalledTimes(1);
+    expect(recordPositionQueryFactory.create).toHaveBeenCalledTimes(2);
+  });
+
+  it('when dryRun is true, should not update position', async () => {
+    objectMetadataService.findManyWithinWorkspace.mockReturnValue([
+      {
+        id: '1',
+        nameSingular: 'company',
+        fields: [],
+      },
+    ]);
+    workspaceDataSourceService.executeRawQuery.mockResolvedValueOnce([
+      {
+        id: '1',
+      },
+    ]);
+    await service.backfill('workspaceId', true);
+    expect(workspaceDataSourceService.executeRawQuery).toHaveBeenCalledTimes(1);
+    expect(recordPositionFactory.create).toHaveBeenCalledTimes(1);
+    expect(recordPositionQueryFactory.create).toHaveBeenCalledTimes(1);
+  });
+});
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-module.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-module.ts
@ -4,9 +4,10 @@ import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/works
 import { RecordPositionQueryFactory } from 'src/engine/api/graphql/workspace-query-builder/factories/record-position-query.factory';
 import { RecordPositionFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/record-position.factory';
 import { RecordPositionBackfillService } from 'src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-service';
+import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';

@Module({
-  imports: [WorkspaceDataSourceModule],
+  imports: [WorkspaceDataSourceModule, ObjectMetadataModule],
  providers: [
    RecordPositionFactory,
    RecordPositionQueryFactory,
--- a/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-service.ts
+++ b/packages/twenty-server/src/engine/api/graphql/workspace-query-runner/services/record-position-backfill-service.ts
@ -1,6 +1,6 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, Logger } from '@nestjs/common';

-import { ObjectMetadataInterface } from 'src/engine/metadata-modules/field-metadata/interfaces/object-metadata.interface';
+import { isDefined } from 'class-validator';

 import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
 import {
@ -8,44 +8,112 @@ import {
  RecordPositionQueryType,
 } from 'src/engine/api/graphql/workspace-query-builder/factories/record-position-query.factory';
 import { RecordPositionFactory } from 'src/engine/api/graphql/workspace-query-runner/factories/record-position.factory';
+import { ObjectMetadataService } from 'src/engine/metadata-modules/object-metadata/object-metadata.service';
+import { hasPositionField } from 'src/engine/metadata-modules/object-metadata/utils/has-position-field.util';

@Injectable()
 export class RecordPositionBackfillService {
+  private readonly logger = new Logger(RecordPositionBackfillService.name);
  constructor(
+    private readonly objectMetadataService: ObjectMetadataService,
    private readonly recordPositionFactory: RecordPositionFactory,
    private readonly recordPositionQueryFactory: RecordPositionQueryFactory,
    private readonly workspaceDataSourceService: WorkspaceDataSourceService,
  ) {}

-  async backfill(
-    workspaceId: string,
-    objectMetadata: { nameSingular: string; isCustom: boolean },
-    recordId: string,
-  ) {
-    const position = await this.recordPositionFactory.create(
-      'last',
-      objectMetadata as ObjectMetadataInterface,
-      workspaceId,
+  async backfill(workspaceId: string, dryRun: boolean) {
+    this.logger.log(
+      `Starting backfilling record positions for workspace ${workspaceId}`,
    );

    const dataSourceSchema =
      this.workspaceDataSourceService.getSchemaName(workspaceId);

-    const [query, params] = await this.recordPositionQueryFactory.create(
-      {
-        recordPositionQueryType: RecordPositionQueryType.UPDATE_POSITION,
-        recordId,
-        positionValue: position,
-      },
-      objectMetadata as ObjectMetadataInterface,
-      dataSourceSchema,
-    );
+    const objectMetadataEntities =
+      await this.objectMetadataService.findManyWithinWorkspace(workspaceId, {
+        where: { isSystem: false },
+      });

-    this.workspaceDataSourceService.executeRawQuery(
-      query,
-      params,
-      workspaceId,
-      undefined,
-    );
+    const objectMetadataWithPosition =
+      objectMetadataEntities.filter(hasPositionField);
+
+    for (const objectMetadata of objectMetadataWithPosition) {
+      const [recordsWithoutPositionQuery, recordsWithoutPositionQueryParams] =
+        this.recordPositionQueryFactory.create(
+          {
+            recordPositionQueryType: RecordPositionQueryType.FIND_BY_POSITION,
+            positionValue: null,
+          },
+          objectMetadata,
+          dataSourceSchema,
+        );
+
+      const recordsWithoutPosition =
+        await this.workspaceDataSourceService.executeRawQuery(
+          recordsWithoutPositionQuery,
+          recordsWithoutPositionQueryParams,
+          workspaceId,
+        );
+
+      if (
+        !isDefined(recordsWithoutPosition) ||
+        recordsWithoutPosition?.length === 0
+      ) {
+        this.logger.log(
+          `No records without position for ${objectMetadata.nameSingular}`,
+        );
+        continue;
+      }
+
+      const position = await this.recordPositionFactory.create(
+        'last',
+        {
+          isCustom: objectMetadata.isCustom,
+          nameSingular: objectMetadata.nameSingular,
+        },
+        workspaceId,
+      );
+
+      for (
+        let recordIndex = 0;
+        recordIndex < recordsWithoutPosition.length;
+        recordIndex++
+      ) {
+        const recordId = recordsWithoutPosition[recordIndex].id;
+
+        if (!recordId) {
+          this.logger.log(
+            `Fetched record without id for ${objectMetadata.nameSingular}`,
+          );
+          continue;
+        }
+
+        const backfilledPosition = position + recordIndex;
+
+        this.logger.log(
+          `Backfilling position ${backfilledPosition} for ${objectMetadata.nameSingular} ${recordId}`,
+        );
+
+        if (dryRun) {
+          continue;
+        }
+
+        const [query, params] = this.recordPositionQueryFactory.create(
+          {
+            recordPositionQueryType: RecordPositionQueryType.UPDATE_POSITION,
+            recordId: recordsWithoutPosition[recordIndex].id,
+            positionValue: position + recordIndex,
+          },
+          objectMetadata,
+          dataSourceSchema,
+        );
+
+        await this.workspaceDataSourceService.executeRawQuery(
+          query,
+          params,
+          workspaceId,
+        );
+      }
+    }
  }
 }