refactor webhookAnalytics call and enrich analytics module (#8253)

**TLDR**

Refactor WebhoonAnalytics Graph to a more abstract version
AnalyticsGraph (in analytics module). Thus enabling the components to be
used on different instances (ex: new endpoint, new kind of graph).

**In order to test:**

1. Set ANALYTICS_ENABLED to true
2. Set TINYBIRD_JWT_TOKEN to the ADMIN token from the workspace
twenty_analytics_playground
3. Set TINYBIRD_JWT_TOKEN to the datasource or your admin token from the
workspace twenty_analytics_playground
4. Create a Webhook in twenty and set wich events it needs to track
5. Run twenty-worker in order to make the webhooks work.
6. Do your tasks in order to populate the data
7. Enter to settings> webhook>your webhook and the statistics section
should be displayed.

---------

Co-authored-by: Félix Malfait <felix@twenty.com>
This commit is contained in:
Ana Sofia Marin Alexandre
2024-11-08 06:00:51 -03:00
committed by GitHub
parent f9c076df31
commit f06cdbdfc6
62 changed files with 1429 additions and 539 deletions

View File

@ -0,0 +1,56 @@
import { computeAnalyticsGraphDataFunction } from '@/analytics/utils/computeAnalyticsGraphDataFunction';
import { mapServerlessFunctionDurationToNivoLineInput } from '@/analytics/utils/mapServerlessFunctionDurationToNivoLineInput';
import { mapServerlessFunctionErrorsToNivoLineInput } from '@/analytics/utils/mapServerlessFunctionErrorsToNivoLineInput';
import { mapWebhookAnalyticsResultToNivoLineInput } from '@/analytics/utils/mapWebhookAnalyticsResultToNivoLineInput';
jest.mock('@/analytics/utils/mapServerlessFunctionDurationToNivoLineInput');
jest.mock('@/analytics/utils/mapServerlessFunctionErrorsToNivoLineInput');
jest.mock('@/analytics/utils/mapWebhookAnalyticsResultToNivoLineInput');
describe('computeAnalyticsGraphDataFunction', () => {
afterEach(() => {
jest.clearAllMocks();
});
it('should return the mapWebhookAnalyticsResultToNivoLineInput function for "getWebhookAnalytics"', () => {
const result = computeAnalyticsGraphDataFunction('getWebhookAnalytics');
expect(result).toBe(mapWebhookAnalyticsResultToNivoLineInput);
});
it('should return the mapServerlessFunctionDurationToNivoLineInput function for "getServerlessFunctionDuration"', () => {
const result = computeAnalyticsGraphDataFunction(
'getServerlessFunctionDuration',
);
expect(result).toBe(mapServerlessFunctionDurationToNivoLineInput);
});
it('should return a function that calls mapServerlessFunctionErrorsToNivoLineInput with "ErrorCount" for "getServerlessFunctionErrorCount"', () => {
const result = computeAnalyticsGraphDataFunction(
'getServerlessFunctionErrorCount',
);
const data = [{ start: '2023-01-01', error_count: 10 }];
result(data);
expect(mapServerlessFunctionErrorsToNivoLineInput).toHaveBeenCalledWith(
data,
'ErrorCount',
);
});
it('should return a function that calls mapServerlessFunctionErrorsToNivoLineInput with "SuccessRate" for "getServerlessFunctionSuccessRate"', () => {
const result = computeAnalyticsGraphDataFunction(
'getServerlessFunctionSuccessRate',
);
const data = [{ start: '2023-01-01', success_rate: 90 }];
result(data);
expect(mapServerlessFunctionErrorsToNivoLineInput).toHaveBeenCalledWith(
data,
'SuccessRate',
);
});
it('should throw an error for an unknown endpoint', () => {
expect(() => computeAnalyticsGraphDataFunction('unknown')).toThrowError(
'No analytics function found associated with endpoint "unknown"',
);
});
});

View File

@ -0,0 +1,143 @@
import { ANALYTICS_GRAPH_OPTION_MAP } from '@/analytics/constants/AnalyticsGraphOptionMap';
import { computeStartEndDate } from '@/analytics/utils/computeStartEndDate';
import { fetchGraphDataOrThrow } from '@/analytics/utils/fetchGraphDataOrThrow';
// Im going to make this test more contundent later
jest.mock('@/analytics/utils/computeStartEndDate', () => ({
computeStartEndDate: jest.fn(() => ({
start: '2024-01-01',
end: '2024-01-07',
})),
}));
describe('fetchGraphDataOrThrow', () => {
// Setup fetch mock
const mockFetch = jest.fn();
global.fetch = mockFetch;
beforeEach(() => {
jest.clearAllMocks();
});
const mockSuccessResponse = {
data: [
{ timestamp: '2024-01-01', count: 10 },
{ timestamp: '2024-01-02', count: 20 },
],
};
const defaultProps = {
recordId: 'test-123',
windowLength: '7D',
tinybirdJwt: 'test-jwt',
endpointName: 'getWebhookAnalytics',
};
it('should fetch data successfully for webhook type', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: () => Promise.resolve(mockSuccessResponse),
});
const result = await fetchGraphDataOrThrow(defaultProps);
// Verify URL construction
const lastCallArgs = mockFetch.mock.calls[0][0];
expect(lastCallArgs).toContain('webhookId=test-123');
expect(lastCallArgs).toContain('getWebhookAnalytics.json');
// Verify headers
const headers = mockFetch.mock.calls[0][1].headers;
expect(headers.Authorization).toBe('Bearer test-jwt');
// Verify response
expect(result).toEqual(mockSuccessResponse.data);
});
it('should handle different window lengths correctly', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: () => Promise.resolve(mockSuccessResponse),
});
await fetchGraphDataOrThrow({
...defaultProps,
windowLength: '1D',
});
// Verify that correct window length options were used
const lastCallArgs = mockFetch.mock.calls[0][0];
const options = ANALYTICS_GRAPH_OPTION_MAP['1D'];
Object.entries(options).forEach(([key, value]) => {
expect(lastCallArgs).toContain(`${key}=${value}`);
});
});
it('should throw error on failed request', async () => {
mockFetch.mockResolvedValueOnce({
ok: false,
json: () => Promise.resolve({ error: 'Failed to fetch' }),
});
await expect(fetchGraphDataOrThrow(defaultProps)).rejects.toThrow(
'Failed to fetch',
);
});
it('should throw error on network failure', async () => {
mockFetch.mockRejectedValueOnce(new Error('Network error'));
await expect(fetchGraphDataOrThrow(defaultProps)).rejects.toThrow(
'Network error',
);
});
it('should use computed start and end dates', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: () => Promise.resolve(mockSuccessResponse),
});
await fetchGraphDataOrThrow(defaultProps);
// Verify computeStartEndDate was called with correct window length
expect(computeStartEndDate).toHaveBeenCalledWith('7D');
// Verify the computed dates are included in the URL
const lastCallArgs = mockFetch.mock.calls[0][0];
expect(lastCallArgs).toContain('start=2024-01-01');
expect(lastCallArgs).toContain('end=2024-01-07');
});
it('should construct URL with all required parameters', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: () => Promise.resolve(mockSuccessResponse),
});
await fetchGraphDataOrThrow(defaultProps);
const lastCallArgs = mockFetch.mock.calls[0][0];
// Check base URL
expect(lastCallArgs).toContain(
'https://api.eu-central-1.aws.tinybird.co/v0/pipes/',
);
// Check endpoint
expect(lastCallArgs).toContain('getWebhookAnalytics.json');
// Check window length options
const options = ANALYTICS_GRAPH_OPTION_MAP['7D'];
Object.entries(options).forEach(([key, value]) => {
expect(lastCallArgs).toContain(`${key}=${value}`);
});
// Check computed dates
expect(lastCallArgs).toContain('start=2024-01-01');
expect(lastCallArgs).toContain('end=2024-01-07');
// Check record ID
expect(lastCallArgs).toContain('webhookId=test-123');
});
});

View File

@ -0,0 +1,65 @@
import { mapServerlessFunctionDurationToNivoLineInput } from '@/analytics/utils/mapServerlessFunctionDurationToNivoLineInput';
describe('mapServerlessFunctionDurationToNivoLineInput', () => {
it('should convert the serverless function duration result to NivoLineInput format', () => {
const serverlessFunctionDurationResult = [
{
start: '2023-01-01T00:00:00.000Z',
minimum: 100,
maximum: 200,
average: 150,
},
{
start: '2023-01-02T00:00:00.000Z',
minimum: 80,
maximum: 160,
average: 120,
},
{
start: '2023-01-03T00:00:00.000Z',
minimum: 90,
maximum: 180,
average: 135,
},
];
const expected = [
{
id: 'Maximum',
data: [
{ x: new Date('2023-01-01T00:00:00.000Z'), y: 200 },
{ x: new Date('2023-01-02T00:00:00.000Z'), y: 160 },
{ x: new Date('2023-01-03T00:00:00.000Z'), y: 180 },
],
},
{
id: 'Minimum',
data: [
{ x: new Date('2023-01-01T00:00:00.000Z'), y: 100 },
{ x: new Date('2023-01-02T00:00:00.000Z'), y: 80 },
{ x: new Date('2023-01-03T00:00:00.000Z'), y: 90 },
],
},
{
id: 'Average',
data: [
{ x: new Date('2023-01-01T00:00:00.000Z'), y: 150 },
{ x: new Date('2023-01-02T00:00:00.000Z'), y: 120 },
{ x: new Date('2023-01-03T00:00:00.000Z'), y: 135 },
],
},
];
const result = mapServerlessFunctionDurationToNivoLineInput(
serverlessFunctionDurationResult,
);
expect(result).toEqual(expected);
});
it('should handle an empty serverless function duration result', () => {
const serverlessFunctionDurationResult = [];
const result = mapServerlessFunctionDurationToNivoLineInput(
serverlessFunctionDurationResult,
);
expect(result).toEqual([]);
});
});

View File

@ -0,0 +1,68 @@
import { mapServerlessFunctionErrorsToNivoLineInput } from '@/analytics/utils/mapServerlessFunctionErrorsToNivoLineInput';
describe('mapServerlessFunctionErrorsToNivoLineInput', () => {
it('should map the serverless function result to Nivo line input format for error count', () => {
const serverlessFunctionResult = [
{ start: '2023-01-01', error_count: 10, success_rate: 0.66 },
{ start: '2023-01-02', error_count: 5, success_rate: 0.75 },
{ start: '2023-01-03', error_count: 8, success_rate: 0.69 },
];
const expected = [
{
id: 'Error',
data: [
{ x: new Date('2023-01-01'), y: 10 },
{ x: new Date('2023-01-02'), y: 5 },
{ x: new Date('2023-01-03'), y: 8 },
],
},
];
const result = mapServerlessFunctionErrorsToNivoLineInput(
serverlessFunctionResult,
'ErrorCount',
);
expect(result).toEqual(expected);
});
it('should map the serverless function result to Nivo line input format for success rate', () => {
const serverlessFunctionResult = [
{ start: '2023-01-01', error_count: 10, success_rate: 0.66 },
{ start: '2023-01-02', error_count: 5, success_rate: 0.75 },
{ start: '2023-01-03', error_count: 8, success_rate: 0.69 },
];
const expected = [
{
id: 'Success Rate',
data: [
{ x: new Date('2023-01-01'), y: 0.66 },
{ x: new Date('2023-01-02'), y: 0.75 },
{ x: new Date('2023-01-03'), y: 0.69 },
],
},
];
const result = mapServerlessFunctionErrorsToNivoLineInput(
serverlessFunctionResult,
'SuccessRate',
);
expect(result).toEqual(expected);
});
it('should handle empty input', () => {
const serverlessFunctionResult = [];
const expected = [
{
id: 'Error',
data: [],
},
];
const result = mapServerlessFunctionErrorsToNivoLineInput(
serverlessFunctionResult,
'ErrorCount',
);
expect(result).toEqual(expected);
});
});

View File

@ -0,0 +1,187 @@
import { mapWebhookAnalyticsResultToNivoLineInput } from '@/analytics/utils/mapWebhookAnalyticsResultToNivoLineInput';
describe('mapWebhookAnalyticsResultToNivoLineInput', () => {
it('should correctly map empty array', () => {
const result = mapWebhookAnalyticsResultToNivoLineInput([]);
expect(result).toEqual([]);
});
it('should correctly map single data point', () => {
const input = [
{
start: '2024-01-01T00:00:00Z',
success_count: 10,
failure_count: 5,
},
];
const expected = [
{
id: 'Failed',
data: [
{
x: new Date('2024-01-01T00:00:00Z'),
y: 5,
},
],
},
{
id: 'Succeeded',
data: [
{
x: new Date('2024-01-01T00:00:00Z'),
y: 10,
},
],
},
];
const result = mapWebhookAnalyticsResultToNivoLineInput(input);
expect(result).toEqual(expected);
});
it('should correctly map multiple data points', () => {
const input = [
{
start: '2024-01-01T00:00:00Z',
success_count: 10,
failure_count: 5,
},
{
start: '2024-01-02T00:00:00Z',
success_count: 15,
failure_count: 3,
},
];
const expected = [
{
id: 'Failed',
data: [
{
x: new Date('2024-01-01T00:00:00Z'),
y: 5,
},
{
x: new Date('2024-01-02T00:00:00Z'),
y: 3,
},
],
},
{
id: 'Succeeded',
data: [
{
x: new Date('2024-01-01T00:00:00Z'),
y: 10,
},
{
x: new Date('2024-01-02T00:00:00Z'),
y: 15,
},
],
},
];
const result = mapWebhookAnalyticsResultToNivoLineInput(input);
expect(result).toEqual(expected);
});
it('should handle zero counts', () => {
const input = [
{
start: '2024-01-01T00:00:00Z',
success_count: 0,
failure_count: 0,
},
];
const expected = [
{
id: 'Failed',
data: [
{
x: new Date('2024-01-01T00:00:00Z'),
y: 0,
},
],
},
{
id: 'Succeeded',
data: [
{
x: new Date('2024-01-01T00:00:00Z'),
y: 0,
},
],
},
];
const result = mapWebhookAnalyticsResultToNivoLineInput(input);
expect(result).toEqual(expected);
});
it('should preserve data point order', () => {
const input = [
{
start: '2024-01-02T00:00:00Z',
success_count: 15,
failure_count: 3,
},
{
start: '2024-01-01T00:00:00Z',
success_count: 10,
failure_count: 5,
},
];
const result = mapWebhookAnalyticsResultToNivoLineInput(input);
// Check that dates in data arrays maintain input order
expect(result[0].data[0].x).toEqual(new Date('2024-01-02T00:00:00Z'));
expect(result[0].data[1].x).toEqual(new Date('2024-01-01T00:00:00Z'));
expect(result[1].data[0].x).toEqual(new Date('2024-01-02T00:00:00Z'));
expect(result[1].data[1].x).toEqual(new Date('2024-01-01T00:00:00Z'));
});
it('should handle malformed dates by creating invalid Date objects', () => {
const input = [
{
start: 'invalid-date',
success_count: 10,
failure_count: 5,
},
];
const result = mapWebhookAnalyticsResultToNivoLineInput(input);
expect(result[0].data[0].x.toString()).toBe('Invalid Date');
expect(result[1].data[0].x.toString()).toBe('Invalid Date');
});
it('should maintain consistent structure with mixed data', () => {
const input = [
{
start: '2024-01-01T00:00:00Z',
success_count: 10,
failure_count: 0,
},
{
start: '2024-01-02T00:00:00Z',
success_count: 0,
failure_count: 5,
},
];
const result = mapWebhookAnalyticsResultToNivoLineInput(input);
// Check both lines exist even when one has zero values
expect(result.length).toBe(2);
expect(result[0].data.length).toBe(2);
expect(result[1].data.length).toBe(2);
});
});

View File

@ -0,0 +1,25 @@
import { mapServerlessFunctionDurationToNivoLineInput } from '@/analytics/utils/mapServerlessFunctionDurationToNivoLineInput';
import { mapServerlessFunctionErrorsToNivoLineInput } from '@/analytics/utils/mapServerlessFunctionErrorsToNivoLineInput';
import { mapWebhookAnalyticsResultToNivoLineInput } from '@/analytics/utils/mapWebhookAnalyticsResultToNivoLineInput';
import { AnalyticsTinybirdJwtMap } from '~/generated-metadata/graphql';
export const computeAnalyticsGraphDataFunction = (
endpointName: keyof AnalyticsTinybirdJwtMap,
) => {
switch (endpointName) {
case 'getWebhookAnalytics':
return mapWebhookAnalyticsResultToNivoLineInput;
case 'getServerlessFunctionDuration':
return mapServerlessFunctionDurationToNivoLineInput;
case 'getServerlessFunctionErrorCount':
return (data: { start: string; error_count: number }[]) =>
mapServerlessFunctionErrorsToNivoLineInput(data, 'ErrorCount');
case 'getServerlessFunctionSuccessRate':
return (data: { start: string; success_rate: number }[]) =>
mapServerlessFunctionErrorsToNivoLineInput(data, 'SuccessRate');
default:
throw new Error(
`No analytics function found associated with endpoint "${endpointName}"`,
);
}
};

View File

@ -0,0 +1,30 @@
import { subDays, subHours } from 'date-fns';
export const computeStartEndDate = (
windowLength: '7D' | '1D' | '12H' | '4H',
) => {
const now = new Date(Date.now());
const end = now.toISOString();
switch (windowLength) {
case '7D':
return {
start: subDays(now, 7).toISOString(),
end,
};
case '1D':
return {
start: subDays(now, 1).toISOString(),
end,
};
case '12H':
return {
start: subHours(now, 12).toISOString(),
end,
};
case '4H':
return {
start: subHours(now, 4).toISOString(),
end,
};
}
};

View File

@ -0,0 +1,38 @@
import { ANALYTICS_ENDPOINT_TYPE_MAP } from '@/analytics/constants/AnalyticsEndpointTypeMap';
import { ANALYTICS_GRAPH_OPTION_MAP } from '@/analytics/constants/AnalyticsGraphOptionMap';
import { AnalyticsComponentProps } from '@/analytics/types/AnalyticsComponentProps';
import { computeStartEndDate } from '@/analytics/utils/computeStartEndDate';
type fetchGraphDataOrThrowProps = AnalyticsComponentProps & {
windowLength: '7D' | '1D' | '12H' | '4H';
tinybirdJwt: string;
};
export const fetchGraphDataOrThrow = async ({
recordId,
windowLength,
tinybirdJwt,
endpointName,
}: fetchGraphDataOrThrowProps) => {
const recordType = ANALYTICS_ENDPOINT_TYPE_MAP[endpointName];
const queryString = new URLSearchParams({
...ANALYTICS_GRAPH_OPTION_MAP[windowLength],
...computeStartEndDate(windowLength),
...{ [`${recordType}Id`]: recordId },
}).toString();
const response = await fetch(
`https://api.eu-central-1.aws.tinybird.co/v0/pipes/${endpointName}.json?${queryString}`,
{
headers: {
Authorization: 'Bearer ' + tinybirdJwt,
},
},
);
const result = await response.json();
if (!response.ok) {
throw new Error(result.error);
}
return result.data;
};

View File

@ -0,0 +1,49 @@
import { NivoLineInput } from '@/analytics/types/NivoLineInput';
//DOING: Adding the servelessFunctionDurationGraph to twenty
export const mapServerlessFunctionDurationToNivoLineInput = (
serverlessFunctionDurationResult: {
start: string;
minimum: number;
maximum: number;
average: number;
}[],
): NivoLineInput[] => {
return serverlessFunctionDurationResult
.flatMap((dataRow) => [
{
x: new Date(dataRow.start),
y: dataRow.maximum,
id: 'Maximum',
},
{
x: new Date(dataRow.start),
y: dataRow.minimum,
id: 'Minimum',
},
{
x: new Date(dataRow.start),
y: dataRow.average,
id: 'Average',
},
])
.reduce(
(
acc: NivoLineInput[],
{ id, x, y }: { id: string; x: Date; y: number },
) => {
const existingGroupIndex = acc.findIndex((group) => group.id === id);
const isExistingGroup = existingGroupIndex !== -1;
if (isExistingGroup) {
return acc.map((group, index) =>
index === existingGroupIndex
? { ...group, data: [...group.data, { x, y }] }
: group,
);
} else {
return [...acc, { id, data: [{ x, y }] }];
}
},
[],
);
};

View File

@ -0,0 +1,26 @@
import { NivoLineInput } from '@/analytics/types/NivoLineInput';
export const mapServerlessFunctionErrorsToNivoLineInput = <
T extends { start: string },
>(
serverlessFunctionResult: (T & {
error_count?: number;
success_rate?: number;
})[],
type: 'ErrorCount' | 'SuccessRate',
): NivoLineInput[] => {
return [
{
id: type === 'ErrorCount' ? 'Error' : 'Success Rate',
data: serverlessFunctionResult.flatMap((dataRow) => [
{
x: new Date(dataRow.start),
y:
type === 'ErrorCount'
? (dataRow.error_count ?? 0)
: (dataRow.success_rate ?? 0),
},
]),
},
];
};

View File

@ -0,0 +1,43 @@
import { NivoLineInput } from '@/analytics/types/NivoLineInput';
export const mapWebhookAnalyticsResultToNivoLineInput = (
webhookAnalyticsResult: {
start: string;
failure_count: number;
success_count: number;
}[],
): NivoLineInput[] => {
return webhookAnalyticsResult
.flatMap((dataRow) => [
{
x: new Date(dataRow.start),
y: dataRow.failure_count,
id: 'Failed',
},
{
x: new Date(dataRow.start),
y: dataRow.success_count,
id: 'Succeeded',
},
])
.reduce(
(
acc: NivoLineInput[],
{ id, x, y }: { id: string; x: Date; y: number },
) => {
const existingGroupIndex = acc.findIndex((group) => group.id === id);
const isExistingGroup = existingGroupIndex !== -1;
if (isExistingGroup) {
return acc.map((group, index) =>
index === existingGroupIndex
? { ...group, data: [...group.data, { x, y }] }
: group,
);
} else {
return [...acc, { id, data: [{ x, y }] }];
}
},
[],
);
};