From f4bc0c687eb77a7470f139433d21cf4fd27a5cb0 Mon Sep 17 00:00:00 2001
From: Weiko <corentin@twenty.com>
Date: Thu, 10 Oct 2024 15:30:13 +0200
Subject: [PATCH] [search] simplify tsvector generated expression and remove
 deletedAt condition (#7561)

## Context
Because we can't create GIN indexes on primitive types and in this case
both tsvector and deletedAt (date), we had to create an index on the
tsvector only and had to make sure it wouldn't return results for
deleted records.
To handle this, we added a CASE in the generated expression to generate
an empty value if the row has been soft-deleted. This is a bit too
complex and won't allow search on softDeleted records in the future.

What we want to do is to make sure deleted records are not returned by
default by adding a WHERE clause in the search and still keep good
performances by also adding a BTREE index on deletedAt column.

When this was implemented, soft-deleted was not handled properly but now
typeorm query builder exclude soft deleted records by default which
means the WHERE clause is not necessary. As for the BTREE index on
deletedAt, this should be part on a broader effort to add it to all
tables and can be added in a later PR.

This PR simply updates the complex tsVectorExpression to only return the
expression regardless of the deletedAt column. Search won't return
soft-deleted records as explained above and perfs will be improved in an
upcoming version.

## Test
tested locally with reset db + created a new workspace. Since the
feature is not launched yet no more effort should be made.
<img width="1289" alt="Screenshot 2024-10-10 at 15 16 16"
src="https://github.com/user-attachments/assets/d9a6f06b-d738-4f1d-8053-2fa93bd8b4f8">

## Note
Some issues with optimistic rendering, we need to refresh once a record
has been deleted otherwise it will still be returned by the search
---
 ...get-ts-vectors-column-expression.utils.spec.ts | 15 +--------------
 .../utils/get-ts-vector-column-expression.util.ts | 10 +---------
 2 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts
index 8c7594b49..8703879e5 100644
--- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/__tests__/get-ts-vectors-column-expression.utils.spec.ts
@@ -73,9 +73,7 @@ describe('getTsVectorColumnExpressionFromFields', () => {
     const fields = [nameFullNameField, jobTitleTextField, emailsEmailsField];
     const result = getTsVectorColumnExpressionFromFields(fields);
     const expected = `
-    CASE 
-      WHEN "deletedAt" IS NULL THEN 
-        to_tsvector('simple', COALESCE("nameFirstName", '') || ' ' || COALESCE("nameLastName", '') || ' ' || COALESCE("jobTitle", '') || ' ' || 
+    to_tsvector('simple', COALESCE("nameFirstName", '') || ' ' || COALESCE("nameLastName", '') || ' ' || COALESCE("jobTitle", '') || ' ' || 
       COALESCE(
         replace(
           "emailsPrimaryEmail",
@@ -85,19 +83,8 @@ describe('getTsVectorColumnExpressionFromFields', () => {
         ''
       )
     )
-      ELSE NULL
-    END
   `.trim();
 
     expect(result.trim()).toBe(expected);
   });
-
-  it('should include CASE statement for handling deletedAt', () => {
-    const fields = [nameTextField];
-    const result = getTsVectorColumnExpressionFromFields(fields);
-
-    expect(result).toContain('CASE');
-    expect(result).toContain('WHEN "deletedAt" IS NULL THEN');
-    expect(result).toContain('ELSE NULL');
-  });
 });
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts
index 816c7b64d..83cabf58b 100644
--- a/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util.ts
@@ -23,15 +23,7 @@ export const getTsVectorColumnExpressionFromFields = (
   );
   const concatenatedExpression = columnExpressions.join(" || ' ' || ");
 
-  const tsVectorExpression = `to_tsvector('simple', ${concatenatedExpression})`;
-
-  return `
-    CASE 
-      WHEN "deletedAt" IS NULL THEN 
-        ${tsVectorExpression}
-      ELSE NULL
-    END
-  `;
+  return `to_tsvector('simple', ${concatenatedExpression})`;
 };
 
 const getColumnExpressionsFromField = (