feat: add findAll and findUnique resolver for universal objects (#1576)

* wip: refacto and start creating custom resolver

* feat: findMany & findUnique of a custom entity

* feat: wip pagination

* feat: initial metadata migration

* feat: universal findAll with pagination

* fix: clean small stuff in pagination

* fix: test

* fix: miss file

* feat: rename custom into universal

* feat: create metadata schema in default database

---------

Co-authored-by: Charles Bochet <charles@twenty.com>

server/src/utils/pagination/find-many-cursor-connection.ts

@@ -0,0 +1,162 @@
+import {
+  LessThanOrEqual,
+  MoreThanOrEqual,
+  ObjectLiteral,
+  SelectQueryBuilder,
+} from 'typeorm';
+
+import { IEdge } from './interfaces/edge.interface';
+import { IConnectionArguments } from './interfaces/connection-arguments.interface';
+import { IOptions } from './interfaces/options.interface';
+import { IConnection } from './interfaces/connection.interface';
+import { validateArgs } from './utils/validate-args';
+import { mergeDefaultOptions } from './utils/default-options';
+import {
+  isBackwardPagination,
+  isForwardPagination,
+} from './utils/pagination-direction';
+import { encodeCursor, extractCursorKeyValue } from './utils/cursor';
+
+/**
+ * Override cursors options
+ */
+export async function findManyCursorConnection<
+  Entity extends ObjectLiteral,
+  Record = Entity,
+  Cursor = { id: string },
+  Node = Record,
+  CustomEdge extends IEdge<Node> = IEdge<Node>,
+>(
+  query: SelectQueryBuilder<Entity>,
+  args: IConnectionArguments = {},
+  initialOptions?: IOptions<Entity, Record, Cursor, Node, CustomEdge>,
+): Promise<IConnection<Node, CustomEdge>> {
+  if (!validateArgs(args)) {
+    throw new Error('Should never happen');
+  }
+
+  const options = mergeDefaultOptions(initialOptions);
+  const totalCountQuery = query.clone();
+  const totalCount = await totalCountQuery.getCount();
+  // Only to extract cursor shape
+  const cursorKeys = Object.keys(options.getCursor(undefined) as any);
+
+  let records: Array<Record>;
+  let hasNextPage: boolean;
+  let hasPreviousPage: boolean;
+
+  // Add order by based on the cursor keys
+  for (const key of cursorKeys) {
+    query.addOrderBy(key, 'ASC');
+  }
+
+  if (isForwardPagination(args)) {
+    // Fetch one additional record to determine if there is a next page
+    const take = args.first + 1;
+
+    // Extract cursor map based on the encoded cursor
+    const cursorMap = extractCursorKeyValue(args.after, options);
+    const skip = cursorMap ? 1 : undefined;
+
+    if (cursorMap) {
+      const [keys, values] = cursorMap;
+
+      // Add `cursor` filter in where condition
+      query.andWhere(
+        keys.reduce((acc, key, index) => {
+          return {
+            ...acc,
+            [key]: MoreThanOrEqual(values[index]),
+          };
+        }, {}),
+      );
+    }
+
+    // Add `take` and `skip` to the query
+    query.take(take).skip(skip);
+
+    // Fetch records
+    records = await options.getRecords(query);
+
+    // See if we are "after" another record, indicating a previous page
+    hasPreviousPage = !!args.after;
+
+    // See if we have an additional record, indicating a next page
+    hasNextPage = records.length > args.first;
+
+    // Remove the extra record (last element) from the results
+    if (hasNextPage) records.pop();
+  } else if (isBackwardPagination(args)) {
+    // Fetch one additional record to determine if there is a previous page
+    const take = -1 * (args.last + 1);
+
+    // Extract cursor map based on the encoded cursor
+    const cursorMap = extractCursorKeyValue(args.before, options);
+    const skip = cursorMap ? 1 : undefined;
+
+    if (cursorMap) {
+      const [keys, values] = cursorMap;
+
+      // Add `cursor` filter in where condition
+      query.andWhere(
+        keys.reduce((acc, key, index) => {
+          return {
+            ...acc,
+            [key]: LessThanOrEqual(values[index]),
+          };
+        }, {}),
+      );
+    }
+
+    // Add `take` and `skip` to the query
+    query.take(take).skip(skip);
+
+    // Fetch records
+    records = await options.getRecords(query);
+
+    // See if we are "before" another record, indicating a next page
+    hasNextPage = !!args.before;
+
+    // See if we have an additional record, indicating a previous page
+    hasPreviousPage = records.length > args.last;
+
+    // Remove the extra record (first element) from the results
+    if (hasPreviousPage) records.shift();
+  } else {
+    // Fetch records
+    records = await options.getRecords(query);
+
+    hasNextPage = false;
+    hasPreviousPage = false;
+  }
+
+  // The cursors are always the first & last elements of the result set
+  const startCursor =
+    records.length > 0 ? encodeCursor(records[0], options) : undefined;
+  const endCursor =
+    records.length > 0
+      ? encodeCursor(records[records.length - 1], options)
+      : undefined;
+
+  // Allow the recordToEdge function to return a custom edge type which will be inferred
+  type EdgeExtended = typeof options.recordToEdge extends (
+    record: Record,
+  ) => infer X
+    ? X extends CustomEdge
+      ? X & { cursor: string }
+      : CustomEdge
+    : CustomEdge;
+
+  const edges = records.map((record) => {
+    return {
+      ...options.recordToEdge(record),
+      cursor: encodeCursor(record, options),
+    } as EdgeExtended;
+  });
+
+  return {
+    edges,
+    pageInfo: { hasNextPage, hasPreviousPage, startCursor, endCursor },
+    totalCount,
+  };
+}

server/src/utils/pagination/index.ts

@@ -0,0 +1,2 @@
+export { ConnectionCursor, ConnectionArgs, Paginated } from './paginated';
+export { findManyCursorConnection } from './find-many-cursor-connection';

server/src/utils/pagination/interfaces/connection-arguments.interface.ts

@@ -0,0 +1,15 @@
+export interface IConnectionArguments {
+  first?: number | null;
+  after?: string | null;
+  last?: number | null;
+  before?: string | null;
+}
+
+export type ConnectionArgumentsUnion =
+  | ForwardPaginationArguments
+  | BackwardPaginationArguments
+  | NoPaginationArguments;
+
+export type ForwardPaginationArguments = { first: number; after?: string };
+export type BackwardPaginationArguments = { last: number; before?: string };
+export type NoPaginationArguments = Record<string, unknown>;

server/src/utils/pagination/interfaces/connection-cursor.type.ts

@@ -0,0 +1 @@
+export type ConnectionCursor = string;

server/src/utils/pagination/interfaces/connection.interface.ts

@@ -0,0 +1,8 @@
+import { IEdge } from './edge.interface';
+import { IPageInfo } from './page-info.interface';
+
+export interface IConnection<T, CustomEdge extends IEdge<T> = IEdge<T>> {
+  edges: Array<CustomEdge>;
+  pageInfo: IPageInfo;
+  totalCount: number;
+}

server/src/utils/pagination/interfaces/edge.interface.ts

@@ -0,0 +1,4 @@
+export interface IEdge<T> {
+  cursor: string;
+  node: T;
+}

server/src/utils/pagination/interfaces/find-many-aguments.interface.ts

@@ -0,0 +1,5 @@
+export interface IFindManyArguments<Cursor> {
+  cursor?: Cursor;
+  take?: number;
+  skip?: number;
+}

server/src/utils/pagination/interfaces/options.interface.ts

@@ -0,0 +1,19 @@
+import { GraphQLResolveInfo } from 'graphql';
+import { ObjectLiteral, SelectQueryBuilder } from 'typeorm';
+
+import { IEdge } from './edge.interface';
+
+export interface IOptions<
+  Entity extends ObjectLiteral,
+  Record,
+  Cursor,
+  Node,
+  CustomEdge extends IEdge<Node>,
+> {
+  getRecords?: (args: SelectQueryBuilder<Entity>) => Promise<Record[]>;
+  getCursor?: (record: Record | undefined) => Cursor;
+  encodeCursor?: (cursor: Cursor) => string;
+  decodeCursor?: (cursorString: string) => Cursor;
+  recordToEdge?: (record: Record) => Omit<CustomEdge, 'cursor'>;
+  resolveInfo?: GraphQLResolveInfo | null;
+}

server/src/utils/pagination/interfaces/page-info.interface.ts

@@ -0,0 +1,6 @@
+export interface IPageInfo {
+  hasNextPage: boolean;
+  hasPreviousPage: boolean;
+  startCursor?: string;
+  endCursor?: string;
+}

server/src/utils/pagination/page-info.ts

@@ -0,0 +1,19 @@
+import { Field, ObjectType } from '@nestjs/graphql';
+
+import { IPageInfo } from './interfaces/page-info.interface';
+import { ConnectionCursor } from './interfaces/connection-cursor.type';
+
+@ObjectType({ isAbstract: true })
+export class PageInfo implements IPageInfo {
+  @Field({ nullable: true })
+  public startCursor!: ConnectionCursor;
+
+  @Field({ nullable: true })
+  public endCursor!: ConnectionCursor;
+
+  @Field(() => Boolean)
+  public hasPreviousPage!: boolean;
+
+  @Field(() => Boolean)
+  public hasNextPage!: boolean;
+}

server/src/utils/pagination/paginated.ts

@@ -0,0 +1,80 @@
+import { Type } from '@nestjs/common';
+import { ArgsType, Directive, Field, ObjectType } from '@nestjs/graphql';
+
+import { IsNumber, IsOptional, IsString } from 'class-validator';
+
+import { PageInfo } from './page-info';
+
+import { IConnectionArguments } from './interfaces/connection-arguments.interface';
+import { IConnection } from './interfaces/connection.interface';
+import { IEdge } from './interfaces/edge.interface';
+import { IPageInfo } from './interfaces/page-info.interface';
+
+export type ConnectionCursor = string;
+
+/**
+ * ConnectionArguments
+ */
+@ArgsType()
+export class ConnectionArgs implements IConnectionArguments {
+  @Field({ nullable: true, description: 'Paginate before opaque cursor' })
+  @IsString()
+  @IsOptional()
+  public before?: ConnectionCursor;
+
+  @Field({ nullable: true, description: 'Paginate after opaque cursor' })
+  @IsString()
+  @IsOptional()
+  public after?: ConnectionCursor;
+
+  @Field({ nullable: true, description: 'Paginate first' })
+  @IsNumber()
+  @IsOptional()
+  public first?: number;
+
+  @Field({ nullable: true, description: 'Paginate last' })
+  @IsNumber()
+  @IsOptional()
+  public last?: number;
+}
+
+/**
+ * Paginated graphQL class inheritance
+ */
+export function Paginated<T>(classRef: Type<T>): Type<IConnection<T>> {
+  @ObjectType(`${classRef.name}Edge`, { isAbstract: true })
+  class Edge implements IEdge<T> {
+    public name = `${classRef.name}Edge`;
+
+    @Field({ nullable: true })
+    public cursor!: ConnectionCursor;
+
+    @Field(() => classRef, { nullable: true })
+    @Directive(`@cacheControl(inheritMaxAge: true)`)
+    public node!: T;
+  }
+
+  @ObjectType(`${classRef.name}Connection`, { isAbstract: true })
+  class Connection implements IConnection<T> {
+    public name = `${classRef.name}Connection`;
+
+    @Field(() => [Edge], { nullable: true })
+    @Directive(`@cacheControl(inheritMaxAge: true)`)
+    public edges!: IEdge<T>[];
+
+    @Field(() => PageInfo, { nullable: true })
+    @Directive(`@cacheControl(inheritMaxAge: true)`)
+    public pageInfo!: IPageInfo;
+
+    @Field()
+    totalCount: number;
+  }
+
+  return Connection as Type<IConnection<T>>;
+}
+
+// export const encodeCursor = <Cursor>(cursor: Cursor) =>
+//   Buffer.from(JSON.stringify(cursor)).toString('base64');
+
+// export const decodeCursor = <Cursor>(cursor: string) =>
+//   JSON.parse(Buffer.from(cursor, 'base64').toString('ascii')) as Cursor;

server/src/utils/pagination/utils/cursor.ts

@@ -0,0 +1,54 @@
+import { ObjectLiteral } from 'typeorm';
+
+import { IEdge } from 'src/utils/pagination/interfaces/edge.interface';
+
+import { MergedOptions } from './default-options';
+
+export function decodeCursor<
+  Entity extends ObjectLiteral,
+  Record,
+  Cursor,
+  Node,
+  CustomEdge extends IEdge<Node>,
+>(
+  connectionCursor: string | undefined,
+  options: MergedOptions<Entity, Record, Cursor, Node, CustomEdge>,
+): Cursor | undefined {
+  if (!connectionCursor) {
+    return undefined;
+  }
+
+  return options.decodeCursor(connectionCursor);
+}
+
+export function encodeCursor<
+  Entity extends ObjectLiteral,
+  Record,
+  Cursor,
+  Node,
+  CustomEdge extends IEdge<Node>,
+>(
+  record: Record,
+  options: MergedOptions<Entity, Record, Cursor, Node, CustomEdge>,
+): string {
+  return options.encodeCursor(options.getCursor(record));
+}
+
+export function extractCursorKeyValue<
+  Entity extends ObjectLiteral,
+  Record,
+  Cursor,
+  Node,
+  CustomEdge extends IEdge<Node>,
+>(
+  connectionCursor: string | undefined,
+  options: MergedOptions<Entity, Record, Cursor, Node, CustomEdge>,
+): [string[], unknown[]] | undefined {
+  const cursor = decodeCursor(connectionCursor, options);
+
+  if (!cursor) {
+    return undefined;
+  }
+
+  return [Object.keys(cursor), Object.values(cursor)];
+}

server/src/utils/pagination/utils/default-options.ts

@@ -0,0 +1,42 @@
+import { ObjectLiteral } from 'typeorm';
+
+import { IEdge } from 'src/utils/pagination/interfaces/edge.interface';
+import { IOptions } from 'src/utils/pagination/interfaces/options.interface';
+
+export type MergedOptions<
+  Entity extends ObjectLiteral,
+  Record,
+  Cursor,
+  Node,
+  CustomEdge extends IEdge<Node>,
+> = Required<IOptions<Entity, Record, Cursor, Node, CustomEdge>>;
+
+export function mergeDefaultOptions<
+  Entity extends ObjectLiteral,
+  Record,
+  Cursor,
+  Node,
+  CustomEdge extends IEdge<Node>,
+>(
+  pOptions?: IOptions<Entity, Record, Cursor, Node, CustomEdge>,
+): MergedOptions<Entity, Record, Cursor, Node, CustomEdge> {
+  return {
+    getRecords: async (query) => {
+      return query.getRawMany();
+    },
+    getCursor: (record: Record | undefined) =>
+      ({ id: (record as unknown as { id: string })?.id } as unknown as Cursor),
+    encodeCursor: (cursor: Cursor) =>
+      Buffer.from((cursor as unknown as { id: string }).id.toString()).toString(
+        'base64',
+      ),
+    decodeCursor: (cursorString: string) =>
+      ({
+        id: Buffer.from(cursorString, 'base64').toString(),
+      } as unknown as Cursor),
+    recordToEdge: (record: Record) =>
+      ({ node: record } as unknown as Omit<CustomEdge, 'cursor'>),
+    resolveInfo: null,
+    ...pOptions,
+  };
+}

server/src/utils/pagination/utils/pagination-direction.ts

@@ -0,0 +1,17 @@
+import {
+  BackwardPaginationArguments,
+  ConnectionArgumentsUnion,
+  ForwardPaginationArguments,
+} from 'src/utils/pagination/interfaces/connection-arguments.interface';
+
+export function isForwardPagination(
+  args: ConnectionArgumentsUnion,
+): args is ForwardPaginationArguments {
+  return 'first' in args && args.first != null;
+}
+
+export function isBackwardPagination(
+  args: ConnectionArgumentsUnion,
+): args is BackwardPaginationArguments {
+  return 'last' in args && args.last != null;
+}

server/src/utils/pagination/utils/validate-args.ts

@@ -0,0 +1,38 @@
+import {
+  ConnectionArgumentsUnion,
+  IConnectionArguments,
+} from 'src/utils/pagination/interfaces/connection-arguments.interface';
+
+export function validateArgs(
+  args: IConnectionArguments,
+): args is ConnectionArgumentsUnion {
+  // Only one of `first` and `last` / `after` and `before` can be set
+  if (args.first != null && args.last != null) {
+    throw new Error('Only one of "first" and "last" can be set');
+  }
+
+  if (args.after != null && args.before != null) {
+    throw new Error('Only one of "after" and "before" can be set');
+  }
+
+  // If `after` is set, `first` has to be set
+  if (args.after != null && args.first == null) {
+    throw new Error('"after" needs to be used with "first"');
+  }
+
+  // If `before` is set, `last` has to be set
+  if (args.before != null && args.last == null) {
+    throw new Error('"before" needs to be used with "last"');
+  }
+
+  // `first` and `last` have to be positive
+  if (args.first != null && args.first <= 0) {
+    throw new Error('"first" has to be positive');
+  }
+
+  if (args.last != null && args.last <= 0) {
+    throw new Error('"last" has to be positive');
+  }
+
+  return true;
+}

server/src/utils/snake-case.ts

@@ -0,0 +1,26 @@
+import isObject from 'lodash.isobject';
+import lodashSnakeCase from 'lodash.snakecase';
+import { SnakeCase, SnakeCasedPropertiesDeep } from 'type-fest';
+
+export const snakeCase = <T>(text: T) =>
+  lodashSnakeCase(text as unknown as string) as SnakeCase<T>;
+
+export const snakeCaseDeep = <T>(value: T): SnakeCasedPropertiesDeep<T> => {
+  // Check if it's an array
+  if (Array.isArray(value)) {
+    return value.map(snakeCaseDeep) as SnakeCasedPropertiesDeep<T>;
+  }
+
+  // Check if it's an object
+  if (isObject(value)) {
+    const result: Record<string, any> = {};
+
+    for (const key in value) {
+      result[snakeCase(key)] = snakeCaseDeep(value[key]);
+    }
+
+    return result as SnakeCasedPropertiesDeep<T>;
+  }
+
+  return value as SnakeCasedPropertiesDeep<T>;
+};