refactor webhookAnalytics call and enrich analytics module (#8253)

**TLDR** Refactor WebhoonAnalytics Graph to a more abstract version AnalyticsGraph (in analytics module). Thus enabling the components to be used on different instances (ex: new endpoint, new kind of graph). **In order to test:** 1. Set ANALYTICS_ENABLED to true 2. Set TINYBIRD_JWT_TOKEN to the ADMIN token from the workspace twenty_analytics_playground 3. Set TINYBIRD_JWT_TOKEN to the datasource or your admin token from the workspace twenty_analytics_playground 4. Create a Webhook in twenty and set wich events it needs to track 5. Run twenty-worker in order to make the webhooks work. 6. Do your tasks in order to populate the data 7. Enter to settings> webhook>your webhook and the statistics section should be displayed. --------- Co-authored-by: Félix Malfait <felix@twenty.com>
2024-11-08 06:00:51 -03:00
parent f9c076df31
commit f06cdbdfc6
62 changed files with 1429 additions and 539 deletions
--- a/packages/twenty-tinybird/README.md
+++ b/packages/twenty-tinybird/README.md
@ -23,6 +23,12 @@ Copy the "admin your@email" token from from https://app.tinybird.co/tokens and p
 ** Auth successful!
 ** Configuration written to .tinyb file, consider adding it to .gitignore
 ```
+You can also log in using your twenty_analytics_token without passing into the interactive mode:
+```sh
+tb auth --token <your twenty_analytics_token >
+** Auth successful!
+** Configuration written to .tinyb file, consider adding it to .gitignore
+```
 To sync your changes to Tinybird use:
 ```sh
 tb push --force --push-deps
--- a/packages/twenty-tinybird/datasources/event.datasource
+++ b/packages/twenty-tinybird/datasources/event.datasource
@ -1,11 +1,11 @@
 SCHEMA >
-    `action` String `json:$.action`,
+    `action` LowCardinality(String) `json:$.action`,
    `timestamp` DateTime64(3) `json:$.timestamp`,
-    `version` String `json:$.version`,
+    `version` LowCardinality(String) `json:$.version`,
    `userId` String `json:$.userId` DEFAULT '',
    `workspaceId` String `json:$.workspaceId` DEFAULT '',
    `payload` String `json:$.payload`

 ENGINE MergeTree
 ENGINE_PARTITION_KEY toYear(timestamp)
-ENGINE_SORTING_KEY action, timestamp
+ENGINE_SORTING_KEY action, workspaceId, timestamp
--- a/packages/twenty-tinybird/datasources/pageview.datasource
+++ b/packages/twenty-tinybird/datasources/pageview.datasource
@ -1,16 +1,16 @@
 SCHEMA >
    `href` String `json:$.href`,
-    `locale` String `json:$.locale`,
+    `locale` LowCardinality(String) `json:$.locale`,
    `pathname` String `json:$.pathname`,
    `referrer` String `json:$.referrer`,
    `sessionId` String `json:$.sessionId`,
-    `timeZone` String `json:$.timeZone`,
+    `timeZone` LowCardinality(String) `json:$.timeZone`,
    `timestamp` DateTime64(3) `json:$.timestamp`,
    `userAgent` String `json:$.userAgent`,
    `userId` String `json:$.userId` DEFAULT '',
-    `version` String `json:$.version`,
-    `workspaceId` String `json:$.workspaceId`
+    `version` LowCardinality(String) `json:$.version`,
+    `workspaceId` String `json:$.workspaceId` DEFAULT ''

 ENGINE MergeTree
 ENGINE_PARTITION_KEY toYear(timestamp)
-ENGINE_SORTING_KEY timestamp, userId, version, workspaceId
+ENGINE_SORTING_KEY workspaceId, userId, timestamp
--- a/packages/twenty-tinybird/datasources/serverlessFunctionEventMV.datasource
+++ b/packages/twenty-tinybird/datasources/serverlessFunctionEventMV.datasource
@ -4,8 +4,9 @@ SCHEMA >
    `functionId` String,
    `durationInMs` Int64,
    `success` Bool,
-    `errorType` String
+    `errorType` LowCardinality(String),
+    `version` LowCardinality(String)

 ENGINE MergeTree
 ENGINE_PARTITION_KEY toYYYYMM(timestamp)
-ENGINE_SORTING_KEY timestamp, functionId, success
+ENGINE_SORTING_KEY workspaceId, functionId, timestamp
--- a/packages/twenty-tinybird/datasources/webhookEventMV.datasource
+++ b/packages/twenty-tinybird/datasources/webhookEventMV.datasource
@ -3,11 +3,11 @@ SCHEMA >
    `workspaceId` String,
    `webhookId` String,
    `url` String,
-    `success` UInt8,
+    `success` Bool,
    `status` Int64,
-    `eventName` String,
-    `version` String
+    `eventName` LowCardinality(String),
+    `version` LowCardinality(String)

 ENGINE MergeTree
 ENGINE_PARTITION_KEY toYYYYMM(timestamp)
-ENGINE_SORTING_KEY timestamp, workspaceId
+ENGINE_SORTING_KEY workspaceId, webhookId, timestamp
--- a/packages/twenty-tinybird/includes/timeSeries.incl
+++ b/packages/twenty-tinybird/includes/timeSeries.incl
@ -6,18 +6,14 @@ SQL >
    %
    WITH
        toStartOfDay(
-            toDateTime64({{ DateTime64(start, '2024-10-16 00:00:00.000') }}, 3),
-            {{ String(timezone, 'UTC') }}
+            parseDateTime64BestEffort({{ String(start, '2024-11-01T00:00:00.000Z') }}, 3)
        ) AS start,
-        toStartOfDay(
-            toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3),
-            {{ String(timezone, 'UTC') }}
-        ) AS
+        toStartOfDay(parseDateTime64BestEffort({{ String(end, '2024-11-02T00:00:00.000Z') }}, 3)) AS
    end
    SELECT
        arrayJoin(
            arrayMap(
-                x -> toDateTime64(toStartOfDay(toDateTime64(x, 3), {{ String(timezone, 'UTC') }}), 3),
+                x -> toDateTime64(toStartOfDay(toDateTime64(x, 3)), 3),
                range(toUInt32(start + 86400), toUInt32(end + 86400),
                86400
            )
@ -29,13 +25,9 @@ SQL >
    %
    WITH
        toStartOfHour(
-            toDateTime64({{ DateTime64(start, '2024-10-22 00:00:00.000') }}, 3),
-            {{ String(timezone, 'UTC') }}
+            parseDateTime64BestEffort({{ String(start, '2024-11-01T00:00:00.000Z') }}, 3)
        ) AS start,
-        toStartOfHour(
-            toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3),
-            {{ String(timezone, 'UTC') }}
-        ) AS
+        toStartOfHour(parseDateTime64BestEffort({{ String(end, '2024-11-02T00:00:00.000Z') }}, 3)) AS
    end
    SELECT
        arrayJoin(
@ -43,45 +35,7 @@ SQL >
        )
    ) as interval

-NODE customIntervals
-SQL >
-    %
-    WITH
-        time_series AS (
-            SELECT
-                toDateTime64(
-                    toDateTime(
-                        toStartOfInterval(
-                            toDateTime64({{ DateTime64(start, '2024-10-22 00:00:00.000') }}, 3),
-                            INTERVAL {{ Int32(tickIntervalInMinutes, 420) }} MINUTE
-                        )
-                    )
-                    + INTERVAL number * {{ Int32(tickIntervalInMinutes, 420) }} MINUTE,
-                    3
-                ) AS interval
-            FROM
-                numbers(
-                    0,
-                    1 + intDiv(
-                        dateDiff(
-                            'minute',
-                            toDateTime64({{ DateTime64(start, '2024-10-22 00:00:00.000') }}, 3),
-                            toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3)
-                        ),
-                        {{ Int32(tickIntervalInMinutes, 420) }}
-                    )
-                )
-            WHERE interval <= toDateTime64({{ DateTime64(end, '2024-10-23 00:00:00.000') }}, 3)
-        )
-    SELECT interval
-    FROM time_series
-
 NODE selectIntervalByGranularity
 SQL >
    %
-    SELECT *
-    FROM
-        {% if granularity == "custom" %} customIntervals
-        {% elif granularity == "hour" %} hourIntervals
-        {% else %} dayIntervals
-        {% end %}
+    SELECT * FROM {% if granularity == "hour" %} hourIntervals {% else %} dayIntervals {% end %}
--- a/packages/twenty-tinybird/pipes/getServerlessFunctionDuration.pipe
+++ b/packages/twenty-tinybird/pipes/getServerlessFunctionDuration.pipe
@ -4,14 +4,8 @@ NODE timeSeriesServerlessFunctionDurationData
 SQL >
    %
    SELECT
-        {% if granularity == "hour" %} toStartOfHour(timestamp, {{ String(timezone, 'UTC') }})
-        {% elif granularity == "custom" %}
-            toDateTime64(
-                toStartOfMinute(timestamp, {{ String(timezone, 'UTC') }}),
-                3,
-                {{ String(timezone, 'UTC') }}
-            )
-        {% else %} toDateTime64(toStartOfDay(timestamp, {{ String(timezone, 'UTC') }}), 3)
+        {% if granularity == "hour" %} toStartOfHour(timestamp)
+        {% else %} toDateTime64(toStartOfDay(timestamp), 3)
        {% end %} AS interval,
        avg(CAST(durationInMs AS Float64)) as average,
        min(durationInMs) as minimum,
@ -19,11 +13,11 @@ SQL >
    FROM serverlessFunctionEventMV
    WHERE
        true
-        AND functionId = {{ String(functionId, 'a9fd87c0-af86-4e17-be3a-a6d3d961678a', required=True) }}
        AND workspaceId
        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
-        AND timestamp >= {{ DateTime(start, '2024-10-22 00:00:00') }}
-        AND timestamp < {{ DateTime(end, '2024-10-23 00:00:00') }}
+        AND functionId = {{ String(functionId, 'a9fd87c0-af86-4e17-be3a-a6d3d961678a', required=True) }}
+        AND timestamp >= parseDateTime64BestEffort({{ String(start, '2024-11-01T00:00:00.000Z') }}, 3)
+        AND timestamp < parseDateTime64BestEffort({{ String(end, '2024-11-02T00:00:00.000Z') }}, 3)
    GROUP BY interval
    ORDER BY interval

--- a/packages/twenty-tinybird/pipes/getServerlessFunctionErrorCount.pipe
+++ b/packages/twenty-tinybird/pipes/getServerlessFunctionErrorCount.pipe
@ -0,0 +1,28 @@
+INCLUDE "../includes/timeSeries.incl"
+-- I decided to separate the error count and the success rate because I think we should maintain the bijection
+-- between an endpoint and a graph in the front-end.
+NODE timeSeriesServerlessFunctionErrorCountData
+SQL >
+    %
+    SELECT
+        {% if granularity == "hour" %} toStartOfHour(timestamp)
+        {% else %} toDateTime64(toStartOfDay(timestamp), 3)
+        {% end %} AS interval,
+        uniqIf(*, success = false) as error_count
+    FROM serverlessFunctionEventMV
+    WHERE
+        true
+        AND workspaceId
+        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
+        AND functionId = {{ String(functionId, 'ad018fc5-eace-4f7e-942f-929560a16459', required=True) }}
+        AND timestamp >= parseDateTime64BestEffort({{ String(start, '2024-11-01T00:00:00.000Z') }}, 3)
+        AND timestamp < parseDateTime64BestEffort({{ String(end, '2024-11-02T00:00:00.000Z') }}, 3)
+    GROUP BY interval
+    ORDER BY interval
+
+NODE endpoint
+SQL >
+    %
+    SELECT formatDateTime(interval, '%FT%T.000%z') as start, error_count
+    FROM selectIntervalByGranularity
+    LEFT JOIN timeSeriesServerlessFunctionErrorCountData USING interval
--- a/packages/twenty-tinybird/pipes/getServerlessFunctionErrors.pipe
+++ b/packages/twenty-tinybird/pipes/getServerlessFunctionErrors.pipe
@ -1,41 +0,0 @@
-INCLUDE "../includes/timeSeries.incl"
-
-NODE timeSeriesServerlessFunctionErrorsData
-SQL >
-    %
-    SELECT
-        {% if granularity == "hour" %} toStartOfHour(timestamp, {{ String(timezone, 'UTC') }})
-        {% elif granularity == "custom" %}
-            toDateTime64(
-                toStartOfMinute(timestamp, {{ String(timezone, 'UTC') }}),
-                3,
-                {{ String(timezone, 'UTC') }}
-            )
-        {% else %} toDateTime64(toStartOfDay(timestamp, {{ String(timezone, 'UTC') }}), 3)
-        {% end %} AS interval,
-        uniqIf(*, success = false) as error_count,
-        round(
-            if(
-                uniqIf(*, success = true) = 0,
-                0,
-                (uniqIf(*, success = true) - uniqIf(*, success = false)) / uniqIf(*, success = true)
-            ),
-            2
-        ) as success_rate
-    FROM serverlessFunctionEventMV
-    WHERE
-        true
-        AND functionId = {{ String(functionId, 'a9fd87c0-af86-4e17-be3a-a6d3d961678a', required=True) }}
-        AND workspaceId
-        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
-        AND timestamp >= {{ DateTime(start, '2024-10-22 00:00:00') }}
-        AND timestamp < {{ DateTime(end, '2024-10-23 00:00:00') }}
-    GROUP BY interval
-    ORDER BY interval
-
-NODE endpoint
-SQL >
-    %
-    SELECT formatDateTime(interval, '%FT%T.000%z') as start, error_count, success_rate
-    FROM selectIntervalByGranularity
-    LEFT JOIN timeSeriesServerlessFunctionErrorsData USING interval
--- a/packages/twenty-tinybird/pipes/getServerlessFunctionSuccessRate.pipe
+++ b/packages/twenty-tinybird/pipes/getServerlessFunctionSuccessRate.pipe
@ -0,0 +1,34 @@
+INCLUDE "../includes/timeSeries.incl"
+
+NODE timeSeriesServerlessFunctionSuccessRateData
+SQL >
+    %
+    SELECT
+        {% if granularity == "hour" %} toStartOfHour(timestamp)
+        {% else %} toDateTime64(toStartOfDay(timestamp), 3)
+        {% end %} AS interval,
+        round(
+            if(
+                uniqIf(*, success = true) = 0,
+                0,
+                (uniqIf(*, success = true) - uniqIf(*, success = false)) / uniqIf(*, success = true)
+            ),
+            2
+        ) as success_rate
+    FROM serverlessFunctionEventMV
+    WHERE
+        true
+        AND workspaceId
+        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
+        AND functionId = {{ String(functionId, 'ad018fc5-eace-4f7e-942f-929560a16459', required=True) }}
+        AND timestamp >= parseDateTime64BestEffort({{ String(start, '2024-11-01T00:00:00.000Z') }}, 3)
+        AND timestamp < parseDateTime64BestEffort({{ String(end, '2024-11-02T00:00:00.000Z') }}, 3)
+    GROUP BY interval
+    ORDER BY interval
+
+NODE endpoint
+SQL >
+    %
+    SELECT formatDateTime(interval, '%FT%T.000%z') as start, success_rate
+    FROM selectIntervalByGranularity
+    LEFT JOIN timeSeriesServerlessFunctionSuccessRateData USING interval
--- a/packages/twenty-tinybird/pipes/getWebhookAnalytics.pipe
+++ b/packages/twenty-tinybird/pipes/getWebhookAnalytics.pipe
@ -4,25 +4,19 @@ NODE timeSeriesWebhookData
 SQL >
    %
    SELECT
-        {% if granularity == "hour" %} toStartOfHour(timestamp, {{ String(timezone, 'UTC') }})
-        {% elif granularity == "custom" %}
-            toDateTime64(
-                toStartOfMinute(timestamp, {{ String(timezone, 'UTC') }}),
-                3,
-                {{ String(timezone, 'UTC') }}
-            )
-        {% else %} toDateTime64(toStartOfDay(timestamp, {{ String(timezone, 'UTC') }}), 3)
+        {% if granularity == "hour" %} toStartOfHour(timestamp)
+        {% else %} toDateTime64(toStartOfDay(timestamp), 3)
        {% end %} AS interval,
        uniqIf(*, success = true) as success_count,
        uniqIf(*, success = false) as failure_count
    FROM webhookEventMV
    WHERE
        true
-        AND webhookId = {{ String(webhookId, '90f12aed-0276-4bea-bcaa-c21ea2763d7d', required=True) }}
        AND workspaceId
        ={{ String(workspaceId, '20202020-1c25-4d02-bf25-6aeccf7ea419', required=True) }}
-        AND timestamp >= {{ DateTime(start, '2024-10-22 00:00:00') }}
-        AND timestamp < {{ DateTime(end, '2024-10-23 00:00:00') }}
+        AND webhookId = {{ String(webhookId, '5237a3bc-566d-4290-b951-96e91051f968', required=True) }}
+        AND timestamp >= parseDateTime64BestEffort({{ String(start, '2024-11-01T00:00:00.000Z') }}, 3)
+        AND timestamp < parseDateTime64BestEffort({{ String(end, '2024-11-02T00:00:00.000Z') }}, 3)
    GROUP BY interval
    ORDER BY interval

--- a/packages/twenty-tinybird/pipes/materializeServerlessFunctionEvent.pipe
+++ b/packages/twenty-tinybird/pipes/materializeServerlessFunctionEvent.pipe
@ -6,7 +6,8 @@ SQL >
        JSONExtractString(payload, 'functionId') as functionId,
        JSONExtractInt(payload, 'duration') as durationInMs,
        if(JSONExtractString(payload, 'status') = 'SUCCESS', TRUE, FALSE) as success,
-        JSONExtractString(payload, 'errorType') as errorType
+        JSONExtractString(payload, 'errorType') as errorType,
+        version
    FROM event
    WHERE action = 'serverlessFunction.executed'