Create filter action (#11904)

Figma
https://www.figma.com/design/xt8O9mFeLl46C5InWwoMrN/Twenty?node-id=59956-288587&t=Dkp83eigIgb3DO6W-11

Issue
https://github.com/orgs/twentyhq/projects/1/views/3?filterQuery=sprint%3A%40current+assignee%3A%40me&pane=issue&itemId=108202682&issue=twentyhq%7Ccore-team-issues%7C897

- filters will be stored as existing GQL filters. It will avoid
re-building frontend
- `applyFilter` function will take the filter and a JS array in input
and returns the filtered array
- filter action calls the util then returns an empty result and error if
no data in the output array. It will end the workflow gracefully.

Example of action:

```
{
      "id": "9d4aeee9-5b78-4053-9615-c367e901ed71",
      "name": "Filter",
      "type": "FILTER",
      "valid": false,
      "settings": {
        "input": {
          "filter": {
            "employees": {
              "gt": 300,
            },
          }
        }
      }
}
```
This commit is contained in:
Thomas Trompette
2025-05-07 14:52:03 +02:00
committed by GitHub
parent 463dee3fe6
commit e96afe444f
16 changed files with 746 additions and 2 deletions

View File

@ -10,4 +10,7 @@ export enum WorkflowStepExecutorExceptionCode {
SCOPED_WORKSPACE_NOT_FOUND = 'SCOPED_WORKSPACE_NOT_FOUND',
INVALID_STEP_TYPE = 'INVALID_STEP_TYPE',
STEP_NOT_FOUND = 'STEP_NOT_FOUND',
INVALID_STEP_SETTINGS = 'INVALID_STEP_SETTINGS',
INTERNAL_ERROR = 'INTERNAL_ERROR',
FAILED_TO_EXECUTE_STEP = 'FAILED_TO_EXECUTE_STEP',
}

View File

@ -7,6 +7,7 @@ import {
WorkflowStepExecutorExceptionCode,
} from 'src/modules/workflow/workflow-executor/exceptions/workflow-step-executor.exception';
import { CodeWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/code/code.workflow-action';
import { FilterWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/filter.workflow-action';
import { FormWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/form/form.workflow-action';
import { SendEmailWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/mail-sender/send-email.workflow-action';
import { CreateRecordWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/record-crud/create-record.workflow-action';
@ -25,6 +26,7 @@ export class WorkflowExecutorFactory {
private readonly deleteRecordWorkflowAction: DeleteRecordWorkflowAction,
private readonly findRecordsWorkflowAction: FindRecordsWorkflowAction,
private readonly formWorkflowAction: FormWorkflowAction,
private readonly filterWorkflowAction: FilterWorkflowAction,
) {}
get(stepType: WorkflowActionType): WorkflowExecutor {
@ -43,6 +45,8 @@ export class WorkflowExecutorFactory {
return this.findRecordsWorkflowAction;
case WorkflowActionType.FORM:
return this.formWorkflowAction;
case WorkflowActionType.FILTER:
return this.filterWorkflowAction;
default:
throw new WorkflowStepExecutorException(
`Workflow step executor not found for step type '${stepType}'`,

View File

@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { FilterWorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/filter.workflow-action';
@Module({
providers: [FilterWorkflowAction],
exports: [FilterWorkflowAction],
})
export class FilterActionModule {}

View File

@ -0,0 +1,70 @@
import { Injectable } from '@nestjs/common';
import { WorkflowExecutor } from 'src/modules/workflow/workflow-executor/interfaces/workflow-executor.interface';
import {
WorkflowStepExecutorException,
WorkflowStepExecutorExceptionCode,
} from 'src/modules/workflow/workflow-executor/exceptions/workflow-step-executor.exception';
import { WorkflowExecutorInput } from 'src/modules/workflow/workflow-executor/types/workflow-executor-input';
import { WorkflowExecutorOutput } from 'src/modules/workflow/workflow-executor/types/workflow-executor-output.type';
import { isWorkflowFilterAction } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/guards/is-workflow-filter-action.guard';
import { applyFilter } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/utils/apply-filter.util';
import { getPreviousStepOutput } from './utils/get-previous-step-output.util';
@Injectable()
export class FilterWorkflowAction implements WorkflowExecutor {
async execute(input: WorkflowExecutorInput): Promise<WorkflowExecutorOutput> {
const { currentStepId, steps, context } = input;
const step = steps.find((step) => step.id === currentStepId);
if (!step) {
throw new WorkflowStepExecutorException(
'Step not found',
WorkflowStepExecutorExceptionCode.STEP_NOT_FOUND,
);
}
if (!isWorkflowFilterAction(step)) {
throw new WorkflowStepExecutorException(
'Step is not a filter action',
WorkflowStepExecutorExceptionCode.INVALID_STEP_TYPE,
);
}
const { filter } = step.settings.input;
if (!filter) {
throw new WorkflowStepExecutorException(
'Filter is not defined',
WorkflowStepExecutorExceptionCode.INVALID_STEP_SETTINGS,
);
}
const previousStepOutput = getPreviousStepOutput(
steps,
currentStepId,
context,
);
const isPreviousStepOutputArray = Array.isArray(previousStepOutput);
const previousStepOutputArray = isPreviousStepOutputArray
? previousStepOutput
: [previousStepOutput];
const filteredOutput = applyFilter(previousStepOutputArray, filter);
if (filteredOutput.length === 0) {
return {
result: undefined,
};
}
return {
result: isPreviousStepOutputArray ? filteredOutput : filteredOutput[0],
};
}
}

View File

@ -0,0 +1,9 @@
import {
WorkflowAction,
WorkflowActionType,
WorkflowFilterAction,
} from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
export const isWorkflowFilterAction = (
action: WorkflowAction,
): action is WorkflowFilterAction => action.type === WorkflowActionType.FILTER;

View File

@ -0,0 +1,12 @@
import { FilterOperator } from './filter-operator.type';
export type FilterCondition = {
and?: FilterCondition[];
or?: FilterCondition[];
not?: FilterCondition;
[key: string]:
| FilterOperator
| FilterCondition
| FilterCondition[]
| undefined;
};

View File

@ -0,0 +1,12 @@
export type FilterOperator = {
eq?: any;
ne?: any;
gt?: number;
gte?: number;
lt?: number;
lte?: number;
like?: string;
ilike?: string;
in?: any[];
is?: 'NULL' | any;
};

View File

@ -0,0 +1,9 @@
import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';
import { BaseWorkflowActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action-settings.type';
export type WorkflowFilterActionSettings = BaseWorkflowActionSettings & {
input: {
filter: Partial<ObjectRecordFilter>;
};
};

View File

@ -0,0 +1,324 @@
import { applyFilter } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/utils/apply-filter.util';
describe('applyFilter', () => {
const testData = [
{
name: { firstName: 'John', lastName: 'Doe' },
companyId: '20202020-171e-4bcc-9cf7-43448d6fb278',
emails: { primaryEmail: 'john.doe@example.com' },
city: 'New York',
age: 30,
tags: ['developer', 'senior'],
metadata: { lastLogin: '2024-03-15', isActive: true },
},
{
name: { firstName: 'Jane', lastName: 'Smith' },
companyId: '30303030-171e-4bcc-9cf7-43448d6fb278',
emails: { primaryEmail: 'jane.smith@test.com' },
city: null,
age: 25,
tags: ['designer', 'junior'],
metadata: { lastLogin: '2024-03-10', isActive: false },
},
{
name: { firstName: 'Tom', lastName: 'Cruise' },
companyId: '20202020-171e-4bcc-9cf7-43448d6fb278',
emails: { primaryEmail: 'tom.cruise@example.com' },
city: '',
age: 40,
tags: ['manager', 'senior'],
metadata: { lastLogin: '2024-03-20', isActive: true },
},
];
describe('Logical Operators', () => {
it('should handle AND conditions', () => {
const filter = {
and: [{ age: { gt: 25 } }, { metadata: { isActive: { eq: true } } }],
};
const result = applyFilter(testData, filter);
expect(result).toEqual([
testData[0], // John Doe
testData[2], // Tom Cruise
]);
});
it('should handle OR conditions', () => {
const filter = {
or: [{ age: { lt: 30 } }, { city: { is: 'NULL' } }],
};
const result = applyFilter(testData, filter);
expect(result).toEqual([
testData[1], // Jane Smith
]);
});
it('should handle NOT conditions', () => {
const filter = {
not: {
metadata: { isActive: { eq: true } },
},
};
const result = applyFilter(testData, filter);
expect(result).toEqual([
testData[1], // Jane Smith
]);
});
it('should handle complex nested logical operators', () => {
const filter = {
and: [
{
or: [{ age: { gt: 35 } }, { city: { is: 'NULL' } }],
},
{
not: {
metadata: { isActive: { eq: false } },
},
},
],
};
const result = applyFilter(testData, filter);
expect(result).toEqual([
testData[2], // Tom Cruise
]);
});
});
describe('Comparison Operators', () => {
it('should handle eq operator', () => {
const filter = {
age: { eq: 30 },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0]]);
});
it('should handle ne operator', () => {
const filter = {
age: { ne: 30 },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[1], testData[2]]);
});
it('should handle gt operator', () => {
const filter = {
age: { gt: 35 },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[2]]);
});
it('should handle gte operator', () => {
const filter = {
age: { gte: 30 },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0], testData[2]]);
});
it('should handle lt operator', () => {
const filter = {
age: { lt: 30 },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[1]]);
});
it('should handle lte operator', () => {
const filter = {
age: { lte: 30 },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0], testData[1]]);
});
});
describe('String Operators', () => {
it('should handle like operator', () => {
const filter = {
name: {
firstName: { like: 'J%' },
},
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0], testData[1]]);
});
it('should handle ilike operator', () => {
const filter = {
name: {
firstName: { ilike: 'j%' },
},
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0], testData[1]]);
});
it('should handle like with multiple wildcards', () => {
const filter = {
emails: {
primaryEmail: { like: '%@example.com' },
},
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0], testData[2]]);
});
});
describe('Array and Null Operators', () => {
it('should handle in operator', () => {
const filter = {
companyId: {
in: ['20202020-171e-4bcc-9cf7-43448d6fb278'],
},
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0], testData[2]]);
});
it('should handle is NULL operator', () => {
const filter = {
city: { is: 'NULL' },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[1]]);
});
it('should handle array contains', () => {
const filter = {
tags: {
in: ['senior'],
},
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0], testData[2]]);
});
});
describe('Nested Object Conditions', () => {
it('should handle deeply nested conditions', () => {
const filter = {
and: [
{
name: {
firstName: { like: 'J%' },
lastName: { like: 'D%' },
},
},
{
metadata: {
isActive: { eq: true },
lastLogin: { like: '2024-03-15' },
},
},
],
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[0]]);
});
it('should handle multiple levels of nesting', () => {
const filter = {
and: [
{
name: {
firstName: { like: 'T%' },
},
},
{
metadata: {
isActive: { eq: true },
},
},
{
emails: {
primaryEmail: { like: '%@example.com' },
},
},
],
};
const result = applyFilter(testData, filter);
expect(result).toEqual([testData[2]]);
});
});
describe('Edge Cases', () => {
it('should handle empty filter', () => {
const filter = {};
const result = applyFilter(testData, filter);
expect(result).toEqual(testData);
});
it('should handle empty array input', () => {
const filter = {
age: { gt: 30 },
};
const result = applyFilter([], filter);
expect(result).toEqual([]);
});
it('should handle non-existent fields', () => {
const filter = {
nonExistentField: { eq: 'value' },
};
const result = applyFilter(testData, filter);
expect(result).toEqual([]);
});
it('should handle null values in nested objects', () => {
const filter = {
metadata: {
nonExistentField: { is: 'NULL' },
},
};
const result = applyFilter(testData, filter);
expect(result).toEqual(testData);
});
});
});

View File

@ -0,0 +1,66 @@
import { WorkflowStepExecutorException } from 'src/modules/workflow/workflow-executor/exceptions/workflow-step-executor.exception';
import { getPreviousStepOutput } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/utils/get-previous-step-output.util';
import { WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
describe('getPreviousStepOutput', () => {
const mockSteps: WorkflowAction[] = [
{
id: 'step1',
nextStepIds: ['step2'],
} as WorkflowAction,
{
id: 'step2',
nextStepIds: ['step3'],
} as WorkflowAction,
{
id: 'step3',
nextStepIds: undefined,
} as WorkflowAction,
];
const mockContext = {
step1: { data: 'step1 output' },
step2: { data: 'step2 output' },
};
it('should return the previous step output when valid', () => {
const result = getPreviousStepOutput(mockSteps, 'step2', mockContext);
expect(result).toEqual({ data: 'step1 output' });
});
it('should throw an error when there is no previous step', () => {
expect(() =>
getPreviousStepOutput(mockSteps, 'step1', mockContext),
).toThrow(WorkflowStepExecutorException);
});
it('should throw an error when there are multiple previous steps', () => {
const stepsWithMultiplePrevious: WorkflowAction[] = [
{
id: 'step1',
nextStepIds: ['step3'],
} as WorkflowAction,
{
id: 'step2',
nextStepIds: ['step3'],
} as WorkflowAction,
{
id: 'step3',
nextStepIds: undefined,
} as WorkflowAction,
];
expect(() =>
getPreviousStepOutput(stepsWithMultiplePrevious, 'step3', mockContext),
).toThrow(WorkflowStepExecutorException);
});
it('should throw an error when previous step output is not found', () => {
const emptyContext = {};
expect(() =>
getPreviousStepOutput(mockSteps, 'step2', emptyContext),
).toThrow(WorkflowStepExecutorException);
});
});

View File

@ -0,0 +1,172 @@
import { FilterCondition } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/types/filter-condition.type';
import { FilterOperator } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/types/filter-operator.type';
export const applyFilter = <T extends Record<string, any>>(
array: T[],
filter: FilterCondition,
): T[] => {
return array.filter((item) => evaluateFilter(item, filter));
};
const evaluateFilter = (
item: Record<string, any>,
filter: FilterCondition,
): boolean => {
if ('and' in filter) {
return (
filter.and?.every((subFilter) => evaluateFilter(item, subFilter)) ?? false
);
}
if ('or' in filter) {
return (
filter.or?.some((subFilter) => evaluateFilter(item, subFilter)) ?? false
);
}
if ('not' in filter && filter.not) {
return !evaluateFilter(item, filter.not);
}
return Object.entries(filter).every(([field, conditions]) => {
const value = item[field];
if (isOperator(conditions)) {
return evaluateCondition(value, conditions as FilterOperator);
}
if (value === undefined && !hasNullCheck(conditions as FilterCondition)) {
return false;
}
return evaluateNestedConditions(value, conditions as FilterCondition);
});
};
const hasNullCheck = (conditions: FilterCondition): boolean => {
const operator = conditions as FilterOperator;
if ('is' in operator && operator.is === 'NULL') {
return true;
}
return Object.values(conditions).some(
(value) =>
typeof value === 'object' &&
value !== null &&
hasNullCheck(value as FilterCondition),
);
};
const evaluateNestedConditions = (
value: unknown,
conditions: FilterCondition,
): boolean => {
const operator = conditions as FilterOperator;
if ('is' in operator && operator.is === 'NULL') {
return value === null || value === undefined;
}
if (value === null || value === undefined) {
return false;
}
return Object.entries(conditions).every(([field, nestedConditions]) => {
const nestedValue = (value as Record<string, any>)[field];
if (isOperator(nestedConditions)) {
return evaluateCondition(nestedValue, nestedConditions as FilterOperator);
}
return evaluateNestedConditions(
nestedValue,
nestedConditions as FilterCondition,
);
});
};
const evaluateCondition = (value: any, condition: FilterOperator): boolean => {
const [[operator, targetValue]] = Object.entries(condition);
switch (operator) {
case 'eq':
return value === targetValue;
case 'ne':
return value !== targetValue;
case 'gt':
return value > targetValue;
case 'gte':
return value >= targetValue;
case 'lt':
return value < targetValue;
case 'lte':
return value <= targetValue;
case 'like':
return matchesLike(value, targetValue as string);
case 'ilike':
return matchesILike(value, targetValue as string);
case 'in':
if (!Array.isArray(targetValue)) return false;
if (Array.isArray(value)) {
return value.some((v) => targetValue.includes(v));
}
return targetValue.includes(value);
case 'is':
return targetValue === 'NULL'
? value === null || value === undefined
: value === targetValue;
default:
return false;
}
};
const matchesLike = (value: any, pattern: string): boolean => {
if (typeof value !== 'string') {
return false;
}
const regexPattern = pattern.replace(/%/g, '.*').replace(/_/g, '.');
return new RegExp(`^${regexPattern}$`).test(value);
};
const matchesILike = (value: any, pattern: string): boolean => {
if (typeof value !== 'string') {
return false;
}
const regexPattern = pattern.replace(/%/g, '.*').replace(/_/g, '.');
return new RegExp(`^${regexPattern}$`, 'i').test(value);
};
const isOperator = (obj: any): boolean => {
if (typeof obj !== 'object' || obj === null) {
return false;
}
return [
'eq',
'ne',
'gt',
'gte',
'lt',
'lte',
'like',
'ilike',
'in',
'is',
].some((op) => op in obj);
};

View File

@ -0,0 +1,41 @@
import {
WorkflowStepExecutorException,
WorkflowStepExecutorExceptionCode,
} from 'src/modules/workflow/workflow-executor/exceptions/workflow-step-executor.exception';
import { WorkflowAction } from 'src/modules/workflow/workflow-executor/workflow-actions/types/workflow-action.type';
export const getPreviousStepOutput = (
steps: WorkflowAction[],
currentStepId: string,
context: Record<string, any>,
) => {
const previousSteps = steps.filter((step) =>
step?.nextStepIds?.includes(currentStepId),
);
if (previousSteps.length === 0) {
throw new WorkflowStepExecutorException(
'Filter action must have a previous step',
WorkflowStepExecutorExceptionCode.FAILED_TO_EXECUTE_STEP,
);
}
if (previousSteps.length > 1) {
throw new WorkflowStepExecutorException(
'Filter action must have only one previous step',
WorkflowStepExecutorExceptionCode.FAILED_TO_EXECUTE_STEP,
);
}
const previousStep = previousSteps[0];
const previousStepOutput = context[previousStep.id];
if (!previousStepOutput) {
throw new WorkflowStepExecutorException(
'Previous step output not found',
WorkflowStepExecutorExceptionCode.FAILED_TO_EXECUTE_STEP,
);
}
return previousStepOutput;
};

View File

@ -54,6 +54,7 @@ export class CreateRecordWorkflowAction implements WorkflowExecutor {
WorkflowStepExecutorExceptionCode.STEP_NOT_FOUND,
);
}
if (!isWorkflowCreateRecordAction(step)) {
throw new WorkflowStepExecutorException(
'Step is not a create record action',

View File

@ -1,5 +1,6 @@
import { OutputSchema } from 'src/modules/workflow/workflow-builder/workflow-schema/types/output-schema.type';
import { WorkflowCodeActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/code/types/workflow-code-action-settings.type';
import { WorkflowFilterActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/types/workflow-filter-action-settings.type';
import { WorkflowFormActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/form/types/workflow-form-action-settings.type';
import { WorkflowSendEmailActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/mail-sender/types/workflow-send-email-action-settings.type';
import {
@ -28,4 +29,5 @@ export type WorkflowActionSettings =
| WorkflowUpdateRecordActionSettings
| WorkflowDeleteRecordActionSettings
| WorkflowFindRecordsActionSettings
| WorkflowFormActionSettings;
| WorkflowFormActionSettings
| WorkflowFilterActionSettings;

View File

@ -1,4 +1,5 @@
import { WorkflowCodeActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/code/types/workflow-code-action-settings.type';
import { WorkflowFilterActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/types/workflow-filter-action-settings.type';
import { WorkflowFormActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/form/types/workflow-form-action-settings.type';
import { WorkflowSendEmailActionSettings } from 'src/modules/workflow/workflow-executor/workflow-actions/mail-sender/types/workflow-send-email-action-settings.type';
import {
@ -17,6 +18,7 @@ export enum WorkflowActionType {
DELETE_RECORD = 'DELETE_RECORD',
FIND_RECORDS = 'FIND_RECORDS',
FORM = 'FORM',
FILTER = 'FILTER',
}
type BaseWorkflowAction = {
@ -63,6 +65,11 @@ export type WorkflowFormAction = BaseWorkflowAction & {
settings: WorkflowFormActionSettings;
};
export type WorkflowFilterAction = BaseWorkflowAction & {
type: WorkflowActionType.FILTER;
settings: WorkflowFilterActionSettings;
};
export type WorkflowAction =
| WorkflowCodeAction
| WorkflowSendEmailAction
@ -70,4 +77,5 @@ export type WorkflowAction =
| WorkflowUpdateRecordAction
| WorkflowDeleteRecordAction
| WorkflowFindRecordsAction
| WorkflowFormAction;
| WorkflowFormAction
| WorkflowFilterAction;

View File

@ -5,6 +5,7 @@ import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/s
import { WorkflowCommonModule } from 'src/modules/workflow/common/workflow-common.module';
import { WorkflowExecutorFactory } from 'src/modules/workflow/workflow-executor/factories/workflow-executor.factory';
import { CodeActionModule } from 'src/modules/workflow/workflow-executor/workflow-actions/code/code-action.module';
import { FilterActionModule } from 'src/modules/workflow/workflow-executor/workflow-actions/filter/filter-action.module';
import { FormActionModule } from 'src/modules/workflow/workflow-executor/workflow-actions/form/form-action.module';
import { SendEmailActionModule } from 'src/modules/workflow/workflow-executor/workflow-actions/mail-sender/send-email-action.module';
import { RecordCRUDActionModule } from 'src/modules/workflow/workflow-executor/workflow-actions/record-crud/record-crud-action.module';
@ -20,6 +21,7 @@ import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow
FormActionModule,
WorkflowRunModule,
BillingModule,
FilterActionModule,
],
providers: [
WorkflowExecutorWorkspaceService,