Merge branch 'dev' of https://github.com/umami-software/umami into dev

db/clickhouse/migrations/11_add_event_session_data_pivot.sql

@@ -55,56 +55,21 @@ SELECT
 FROM umami.event_data
 GROUP BY website_id, session_id, event_id, event_name, url_path, created_at;
 
-CREATE TABLE IF NOT EXISTS umami.session_data_pivot
-(
-    website_id      UUID,
-    session_id      UUID,
-    distinct_id     String,
-    created_year_month UInt32,
-    created_at      AggregateFunction(max, DateTime('UTC')),
-    property_keys   AggregateFunction(groupArray, String),
-    property_values AggregateFunction(groupArray, String),
-    property_types  AggregateFunction(groupArray, UInt32)
-)
-ENGINE = AggregatingMergeTree()
-PARTITION BY created_year_month
-ORDER BY (website_id, session_id, distinct_id)
-SETTINGS index_granularity = 8192;
+ALTER TABLE umami.session_data
+MODIFY SETTING deduplicate_merge_projection_mode = 'drop';
 
-CREATE MATERIALIZED VIEW IF NOT EXISTS umami.session_data_pivot_mv
-TO umami.session_data_pivot
-AS SELECT
-    website_id,
-    session_id,
-    ifNull(distinct_id, '') AS distinct_id,
-    toYYYYMM(max(session_data.created_at)) AS created_year_month,
-    maxState(session_data.created_at) AS created_at,
-    groupArrayState(data_key) AS property_keys,
-    groupArrayState(multiIf(
-        data_type IN (1, 3, 5), ifNull(string_value, ''),
-        data_type = 2, toString(ifNull(number_value, 0)),
-        data_type = 4, toString(ifNull(date_value, toDateTime(0))),
-        ''
-    )) AS property_values,
-    groupArrayState(data_type) AS property_types
-FROM umami.session_data
-GROUP BY website_id, session_id, distinct_id;
+ALTER TABLE umami.session_data
+ADD PROJECTION session_data_property_filter_projection (
+    SELECT *
+    ORDER BY (
+        website_id,
+        data_key,
+        data_type,
+        string_value,
+        number_value,
+        date_value,
+        session_id
+    )
+);
 
--- Backfill existing session data
-INSERT INTO umami.session_data_pivot
-SELECT
-    website_id,
-    session_id,
-    ifNull(distinct_id, '') AS distinct_id,
-    toYYYYMM(max(session_data.created_at)) AS created_year_month,
-    maxState(session_data.created_at) AS created_at,
-    groupArrayState(data_key),
-    groupArrayState(multiIf(
-        data_type IN (1, 3, 5), ifNull(string_value, ''),
-        data_type = 2, toString(ifNull(number_value, 0)),
-        data_type = 4, toString(ifNull(date_value, toDateTime(0))),
-        ''
-    )),
-    groupArrayState(data_type)
-FROM umami.session_data
-GROUP BY website_id, session_id, distinct_id;
+ALTER TABLE umami.session_data MATERIALIZE PROJECTION session_data_property_filter_projection;

db/clickhouse/schema.sql

@@ -90,6 +90,25 @@ ENGINE = ReplacingMergeTree
     ORDER BY (website_id, session_id, data_key)
     SETTINGS index_granularity = 8192;
 
+ALTER TABLE umami.session_data
+MODIFY SETTING deduplicate_merge_projection_mode = 'drop';
+
+ALTER TABLE umami.session_data
+ADD PROJECTION session_data_property_filter_projection (
+    SELECT *
+    ORDER BY (
+        website_id,
+        data_key,
+        data_type,
+        string_value,
+        number_value,
+        date_value,
+        session_id
+    )
+);
+
+ALTER TABLE umami.session_data MATERIALIZE PROJECTION session_data_property_filter_projection;
+
 -- stats hourly
 CREATE TABLE umami.website_event_stats_hourly
 (