From 843f104c156bca25150c72a9b135437fb109c044 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 11 Mar 2026 10:36:50 -0700 Subject: [PATCH] update event-data to paged query --- .../websites/[websiteId]/event-data/route.ts | 9 +- src/queries/sql/events/getEventData.ts | 115 ++++++++++++++---- 2 files changed, 95 insertions(+), 29 deletions(-) diff --git a/src/app/api/websites/[websiteId]/event-data/route.ts b/src/app/api/websites/[websiteId]/event-data/route.ts index 193dfda82..2d0caf65e 100644 --- a/src/app/api/websites/[websiteId]/event-data/route.ts +++ b/src/app/api/websites/[websiteId]/event-data/route.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { json, unauthorized } from '@/lib/response'; -import { filterParams } from '@/lib/schema'; +import { filterParams, pagingParams } from '@/lib/schema'; import { canViewWebsite } from '@/permissions'; import { getEventData } from '@/queries/sql/events/getEventData'; @@ -13,6 +13,7 @@ export async function GET( startAt: z.coerce.number().int(), endAt: z.coerce.number().int(), ...filterParams, + ...pagingParams, }); const { auth, query, error } = await parseRequest(request, schema); @@ -29,7 +30,7 @@ export async function GET( const filters = await getQueryFilters(query, websiteId); - const rows = await getEventData(websiteId, filters); + const { data: rows, count, page, pageSize } = await getEventData(websiteId, filters); const eventMap = new Map< string, @@ -45,5 +46,5 @@ export async function GET( entry.eventProperties.push(props); } - return json([...eventMap.values()]); -} + return json({ data: [...eventMap.values()], count, page, pageSize }); +} \ No newline at end of file diff --git a/src/queries/sql/events/getEventData.ts b/src/queries/sql/events/getEventData.ts index d6cefbda6..4a3fac860 100644 --- a/src/queries/sql/events/getEventData.ts +++ b/src/queries/sql/events/getEventData.ts @@ -1,4 +1,5 @@ import clickhouse from '@/lib/clickhouse'; +import { DEFAULT_PAGE_SIZE } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; @@ -14,13 +15,42 @@ export async function getEventData(...args: [websiteId: string, filters: QueryFi async function relationalQuery(websiteId: string, filters: QueryFilters) { const { rawQuery, parseFilters } = prisma; + const { page = 1, pageSize } = filters; + const size = +pageSize || DEFAULT_PAGE_SIZE; + const offset = +size * (+page - 1); + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, }); - return rawQuery( + // Selects distinct event IDs matching all filters — reused for count and paged data + const eventQuery = ` + select website_event.event_id + from website_event + join event_data on event_data.website_event_id = website_event.event_id + and event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + ${filterQuery} + group by website_event.event_id + `; + + const count = await rawQuery( + `select count(*) as num from (${eventQuery}) t`, + queryParams, + ).then((res: any) => res[0].num); + + const data = await rawQuery( ` + with paged_events as ( + ${eventQuery} + order by max(website_event.created_at) desc + limit ${size} offset ${offset} + ) select event_data.website_id as "websiteId", event_data.website_event_id as "eventId", @@ -35,53 +65,88 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { join website_event on website_event.event_id = event_data.website_event_id and website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} - ${cohortQuery} - ${joinSessionQuery} + join paged_events on paged_events.event_id = event_data.website_event_id where event_data.website_id = {{websiteId::uuid}} and event_data.created_at between {{startDate}} and {{endDate}} - ${filterQuery} order by event_data.created_at desc - limit 1000 `, queryParams, FUNCTION_NAME, ); + + return { data, count, page: +page, pageSize: size }; } async function clickhouseQuery(websiteId: string, filters: QueryFilters) { const { rawQuery, parseFilters } = clickhouse; + const { page = 1, pageSize } = filters; + const size = +pageSize || DEFAULT_PAGE_SIZE; + const offset = +size * (+page - 1); + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); - return rawQuery( - ` - select - website_id as websiteId, - event_id as eventId, - event_name as eventName, - data_key as dataKey, - string_value as stringValue, - number_value as numberValue, - date_value as dateValue, - data_type as dataType, - created_at as createdAt + // Selects distinct event IDs matching all filters — reused for count and paged data + const eventQuery = ` + select event_data.event_id from event_data any left join ( - select * - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = 2) website_event + select event_id, session_id, website_id, event_name, created_at + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = 2 + ) website_event on website_event.event_id = event_data.event_id and website_event.session_id = event_data.session_id and website_event.website_id = event_data.website_id ${cohortQuery} where event_data.website_id = {websiteId:UUID} and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} - ${filterQuery} + ${filterQuery} + group by event_data.event_id + `; + + const count = await rawQuery( + `select count(*) as num from (${eventQuery}) t`, + queryParams, + ).then((res: any) => res[0].num); + + const data = await rawQuery( + ` + with paged_events as ( + ${eventQuery} + order by max(event_data.created_at) desc + limit ${size} offset ${offset} + ) + select + event_data.website_id as websiteId, + event_data.event_id as eventId, + website_event.event_name as eventName, + data_key as dataKey, + string_value as stringValue, + number_value as numberValue, + date_value as dateValue, + data_type as dataType, + event_data.created_at as createdAt + from event_data + any left join ( + select event_id, session_id, website_id, event_name + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = 2 + ) website_event + on website_event.event_id = event_data.event_id + and website_event.session_id = event_data.session_id + and website_event.website_id = event_data.website_id + inner join paged_events on paged_events.event_id = event_data.event_id + where event_data.website_id = {websiteId:UUID} + and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} order by event_data.created_at desc - limit 1000 `, queryParams, FUNCTION_NAME, ); -} + + return { data, count, page: +page, pageSize: size }; +} \ No newline at end of file