From fbac7518fc35c608a58fe7d55efaa17bba831417 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 4 May 2026 15:43:05 -0700 Subject: [PATCH] Add MAX_PAGING_RESULTS on hooks for performance improvements. --- src/components/common/DataGrid.tsx | 3 +- src/components/common/Pager.tsx | 9 +- .../hooks/queries/useEventDataPivotQuery.ts | 2 + .../hooks/queries/useReplaysQuery.ts | 3 +- .../hooks/queries/useRevenueSessionsQuery.ts | 3 +- .../hooks/queries/useSavedReplaysQuery.ts | 1 - .../hooks/queries/useSessionDataPivotQuery.ts | 2 + .../hooks/queries/useWebsiteEventsQuery.ts | 2 + .../hooks/queries/useWebsiteSessionsQuery.ts | 3 +- src/lib/clickhouse.ts | 10 +- src/lib/constants.ts | 1 + src/lib/prisma.ts | 10 +- src/lib/request.ts | 1 + src/lib/schema.ts | 1 + src/lib/types.ts | 2 + src/queries/sql/events/getEventDataPivot.ts | 132 ++++-------------- .../sql/sessions/getSessionDataPivot.ts | 104 ++------------ 17 files changed, 74 insertions(+), 215 deletions(-) diff --git a/src/components/common/DataGrid.tsx b/src/components/common/DataGrid.tsx index 993929fd2..8d18321cf 100644 --- a/src/components/common/DataGrid.tsx +++ b/src/components/common/DataGrid.tsx @@ -41,7 +41,7 @@ export function DataGrid({ const { data, error, isLoading, isFetching } = query; const { router, updateParams, query: queryParams } = useNavigation(); const [search, setSearch] = useState(queryParams?.search || data?.search || ''); - const showPager = allowPaging && data && data.count > data.pageSize; + const showPager = allowPaging && data && (data.count > data.pageSize || data.isCapped); const { isMobile } = useMobile(); const displayMode = isMobile ? 'cards' : undefined; @@ -95,6 +95,7 @@ export function DataGrid({ page={data.page} pageSize={data.pageSize} count={data.count} + isCapped={data.isCapped} onPageChange={handlePageChange} /> diff --git a/src/components/common/Pager.tsx b/src/components/common/Pager.tsx index 128d33e24..ff22dc70e 100644 --- a/src/components/common/Pager.tsx +++ b/src/components/common/Pager.tsx @@ -6,11 +6,12 @@ export interface PagerProps { page: string | number; pageSize: string | number; count: string | number; + isCapped?: boolean; onPageChange: (nextPage: number) => void; className?: string; } -export function Pager({ page, pageSize, count, onPageChange }: PagerProps) { +export function Pager({ page, pageSize, count, isCapped, onPageChange }: PagerProps) { const { t, labels } = useMessages(); const maxPage = pageSize && count ? Math.ceil(+count / +pageSize) : 0; const lastPage = page === maxPage; @@ -28,13 +29,15 @@ export function Pager({ page, pageSize, count, onPageChange }: PagerProps) { } }; - if (maxPage === 1) { + if (maxPage === 1 && !isCapped) { return null; } + const displayCount = isCapped ? `10,000+` : (+count).toLocaleString(); + return ( - {t(labels.numberOfRecords, { x: count.toLocaleString() })} + {t(labels.numberOfRecords, { x: displayCount })} {t(labels.pageOf, { diff --git a/src/components/hooks/queries/useEventDataPivotQuery.ts b/src/components/hooks/queries/useEventDataPivotQuery.ts index e9f689ac7..a6ef902bf 100644 --- a/src/components/hooks/queries/useEventDataPivotQuery.ts +++ b/src/components/hooks/queries/useEventDataPivotQuery.ts @@ -1,3 +1,4 @@ +import { MAX_PAGING_RESULTS } from '@/lib/constants'; import { serializeEventPropertyFilters } from '@/lib/params'; import type { EventPropertyFilter, ReactQueryOptions } from '@/lib/types'; import { useApi } from '../useApi'; @@ -28,6 +29,7 @@ export function useEventDataPivotQuery( timezone, ...serializeEventPropertyFilters(eventFilters), ...params, + maxResults: MAX_PAGING_RESULTS, }), enabled: !!(websiteId && eventName), ...options, diff --git a/src/components/hooks/queries/useReplaysQuery.ts b/src/components/hooks/queries/useReplaysQuery.ts index 79c1a04bf..795512896 100644 --- a/src/components/hooks/queries/useReplaysQuery.ts +++ b/src/components/hooks/queries/useReplaysQuery.ts @@ -1,3 +1,4 @@ +import { MAX_PAGING_RESULTS } from '@/lib/constants'; import { useApi } from '../useApi'; import { useDateParameters } from '../useDateParameters'; import { useFilterParameters } from '../useFilterParameters'; @@ -24,7 +25,7 @@ export function useReplaysQuery(websiteId: string, params?: Record { return get(`/websites/${websiteId}/replays/saved`, { ...pageParams, - pageSize: 20, }); }, }); diff --git a/src/components/hooks/queries/useSessionDataPivotQuery.ts b/src/components/hooks/queries/useSessionDataPivotQuery.ts index db5fecccf..80a6bee89 100644 --- a/src/components/hooks/queries/useSessionDataPivotQuery.ts +++ b/src/components/hooks/queries/useSessionDataPivotQuery.ts @@ -1,3 +1,4 @@ +import { MAX_PAGING_RESULTS } from '@/lib/constants'; import { serializePropertyFilters } from '@/lib/params'; import type { PropertyFilter, ReactQueryOptions } from '@/lib/types'; import { useApi } from '../useApi'; @@ -28,6 +29,7 @@ export function useSessionDataPivotQuery( propertyName, ...serializePropertyFilters(propertyFilters), ...params, + maxResults: MAX_PAGING_RESULTS, }), enabled: !!(websiteId && propertyName), ...options, diff --git a/src/components/hooks/queries/useWebsiteEventsQuery.ts b/src/components/hooks/queries/useWebsiteEventsQuery.ts index fc4dad5b1..e28e03369 100644 --- a/src/components/hooks/queries/useWebsiteEventsQuery.ts +++ b/src/components/hooks/queries/useWebsiteEventsQuery.ts @@ -1,3 +1,4 @@ +import { MAX_PAGING_RESULTS } from '@/lib/constants'; import type { ReactQueryOptions } from '@/lib/types'; import { useApi } from '../useApi'; import { useDateParameters } from '../useDateParameters'; @@ -32,6 +33,7 @@ export function useWebsiteEventsQuery( ...filters, ...pageParams, eventType: EVENT_TYPES[params.view], + maxResults: MAX_PAGING_RESULTS, }), enabled: !!websiteId, ...options, diff --git a/src/components/hooks/queries/useWebsiteSessionsQuery.ts b/src/components/hooks/queries/useWebsiteSessionsQuery.ts index 31906be90..9de64850c 100644 --- a/src/components/hooks/queries/useWebsiteSessionsQuery.ts +++ b/src/components/hooks/queries/useWebsiteSessionsQuery.ts @@ -1,3 +1,4 @@ +import { MAX_PAGING_RESULTS } from '@/lib/constants'; import { useApi } from '../useApi'; import { useDateParameters } from '../useDateParameters'; import { useFilterParameters } from '../useFilterParameters'; @@ -27,7 +28,7 @@ export function useWebsiteSessionsQuery( ...filters, ...pageParams, ...params, - pageSize: 20, + maxResults: MAX_PAGING_RESULTS, }); }, }); diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 025e4273d..ae44c9147 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -380,13 +380,15 @@ async function pagedRawQuery( .filter(n => n) .join('\n'); - const count = await rawQuery(`select count(*) as num from (${query}) t`, queryParams).then( - res => res[0].num, - ); + const { maxResults } = filters; + const countQuery = maxResults + ? `select count(*) as num from (select 1 from (${query}) t limit ${+maxResults}) t2` + : `select count(*) as num from (${query}) t`; + const count = await rawQuery(countQuery, queryParams).then(res => res[0].num); const data = await rawQuery(`${query}${statements}`, queryParams, name); - return { data, count, page: +page, pageSize: size, orderBy, search }; + return { data, count, page: +page, pageSize: size, orderBy, search, isCapped: !!maxResults && +count >= +maxResults }; } async function rawQuery( diff --git a/src/lib/constants.ts b/src/lib/constants.ts index f7c182b15..dc01a6d48 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -26,6 +26,7 @@ export const DEFAULT_DATE_RANGE_VALUE = '24hour'; export const DEFAULT_WEBSITE_LIMIT = 10; export const DEFAULT_RESET_DATE = '2000-01-01'; export const DEFAULT_PAGE_SIZE = 20; +export const MAX_PAGING_RESULTS = 10000; export const DEFAULT_DATE_COMPARE = 'prev'; export const DEFAULT_CURRENCY = 'USD'; diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 95d585c1e..7f400d05b 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -470,13 +470,15 @@ async function pagedRawQuery( .filter(n => n) .join('\n'); - const count = await rawQuery(`select count(*) as num from (${query}) t`, queryParams).then( - res => res[0].num, - ); + const { maxResults } = filters; + const countQuery = maxResults + ? `select count(*) as num from (select 1 from (${query}) t limit ${+maxResults}) t2` + : `select count(*) as num from (${query}) t`; + const count = await rawQuery(countQuery, queryParams).then(res => Number(res[0].num)); const data = await rawQuery(`${query}${statements}`, queryParams, name); - return { data, count, page: +page, pageSize: size, orderBy }; + return { data, count, page: +page, pageSize: size, orderBy, isCapped: !!maxResults && +count >= +maxResults }; } function getSearchParameters(query: string, filters: Record[]) { diff --git a/src/lib/request.ts b/src/lib/request.ts index 9b32b6736..87c4872ad 100644 --- a/src/lib/request.ts +++ b/src/lib/request.ts @@ -174,5 +174,6 @@ export async function getQueryFilters( sortDescending: params?.sortDescending, search: params?.search, compare: params?.compare, + maxResults: params?.maxResults, }; } diff --git a/src/lib/schema.ts b/src/lib/schema.ts index 2567c3de6..113b24cf8 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -77,6 +77,7 @@ export const searchParams = { export const pagingParams = { page: z.coerce.number().int().positive().optional(), pageSize: z.coerce.number().int().positive().optional(), + maxResults: z.coerce.number().int().positive().optional(), }; export const sortingParams = { diff --git a/src/lib/types.ts b/src/lib/types.ts index dd365aeab..6c8bb92ff 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -157,6 +157,7 @@ export interface SortParams { export interface PageParams { page?: number; pageSize?: number; + maxResults?: number; } export interface SegmentParams { @@ -172,6 +173,7 @@ export interface PageResult { orderBy?: string; sortDescending?: boolean; search?: string; + isCapped?: boolean; } export interface RealtimeData { diff --git a/src/queries/sql/events/getEventDataPivot.ts b/src/queries/sql/events/getEventDataPivot.ts index 8699d3269..210a7cdc6 100644 --- a/src/queries/sql/events/getEventDataPivot.ts +++ b/src/queries/sql/events/getEventDataPivot.ts @@ -1,5 +1,4 @@ import clickhouse from '@/lib/clickhouse'; -import { DEFAULT_PAGE_SIZE } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { EventPropertyFilter, QueryFilters } from '@/lib/types'; @@ -27,10 +26,7 @@ async function relationalQuery( eventFilters: EventPropertyFilter[] = [], ) { const { timezone = 'utc' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma; - const { page = 1, pageSize } = filters; - const size = +pageSize || DEFAULT_PAGE_SIZE; - const offset = +size * (+page - 1); + const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma; const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, @@ -39,30 +35,10 @@ async function relationalQuery( }); const { sql: pfSQL, params: pfParams } = getPropertyFilterQuery(eventFilters, 'event', timezone); - const countResult = await rawQuery( - ` - select count(distinct website_event.event_id) as num - from website_event - join event_data on event_data.website_event_id = website_event.event_id - and event_data.website_id = {{websiteId::uuid}} - and event_data.created_at between {{startDate}} and {{endDate}} - ${cohortQuery} - ${joinSessionQuery} - where website_event.website_id = {{websiteId::uuid}} - and website_event.created_at between {{startDate}} and {{endDate}} - and website_event.event_name = {{eventName}} - ${filterQuery} - ${pfSQL} - `, - { ...queryParams, eventName, ...pfParams }, - ); - - const count = countResult[0].num; - - const rows = await rawQuery( + return pagedRawQuery( ` with paged_events as ( - select website_event.event_id + select website_event.event_id, max(website_event.created_at) as sort_created_at from website_event join event_data on event_data.website_event_id = website_event.event_id and event_data.website_id = {{websiteId::uuid}} @@ -75,68 +51,43 @@ async function relationalQuery( ${filterQuery} ${pfSQL} group by website_event.event_id - order by max(website_event.created_at) desc - limit ${size} offset ${offset} ) select website_event.event_id as "eventId", website_event.session_id as "sessionId", website_event.event_name as "eventName", website_event.url_path as "urlPath", - website_event.created_at as "createdAt", - event_data.data_key as "dataKey", - coalesce( - case when event_data.data_type = 1 then event_data.string_value end, - case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end, - case when event_data.data_type = 3 then event_data.string_value end, - case when event_data.data_type = 4 then ${getDateStringSQL('event_data.date_value', 'second', timezone)} end, - case when event_data.data_type = 5 then event_data.string_value end, - '' - ) as "value", - event_data.data_type as "dataType" + max(website_event.created_at) as "createdAt", + array_agg(event_data.data_key order by event_data.data_key asc) as "propertyKeys", + array_agg( + coalesce( + case when event_data.data_type = 1 then event_data.string_value end, + case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end, + case when event_data.data_type = 3 then event_data.string_value end, + case when event_data.data_type = 4 then ${getDateStringSQL('event_data.date_value', 'second', timezone)} end, + case when event_data.data_type = 5 then event_data.string_value end, + '' + ) + order by event_data.data_key asc + ) as "propertyValues" from event_data join website_event on website_event.event_id = event_data.website_event_id and website_event.website_id = {{websiteId::uuid}} join paged_events on paged_events.event_id = event_data.website_event_id where event_data.website_id = {{websiteId::uuid}} and event_data.created_at between {{startDate}} and {{endDate}} - order by website_event.created_at desc + group by + website_event.event_id, + website_event.session_id, + website_event.event_name, + website_event.url_path, + paged_events.sort_created_at + order by paged_events.sort_created_at desc `, { ...queryParams, eventName, ...pfParams }, + filters, FUNCTION_NAME, ); - - // Pivot flat rows into one record per event - const eventMap = new Map< - string, - { - eventId: string; - sessionId: string; - eventName: string; - urlPath: string; - createdAt: Date; - propertyKeys: string[]; - propertyValues: string[]; - } - >(); - for (const { eventId, sessionId, eventName: name, urlPath, createdAt, dataKey, value } of rows) { - if (!eventMap.has(eventId)) { - eventMap.set(eventId, { - eventId, - sessionId, - eventName: name, - urlPath, - createdAt, - propertyKeys: [], - propertyValues: [], - }); - } - const entry = eventMap.get(eventId); - entry.propertyKeys.push(dataKey); - entry.propertyValues.push(value ?? ''); - } - - return { data: [...eventMap.values()], count, page: +page, pageSize: size }; } async function clickhouseQuery( @@ -146,10 +97,7 @@ async function clickhouseQuery( eventFilters: EventPropertyFilter[] = [], ) { const { timezone = 'UTC' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse; - const { page = 1, pageSize } = filters; - const size = +pageSize || DEFAULT_PAGE_SIZE; - const offset = +size * (+page - 1); + const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -158,31 +106,7 @@ async function clickhouseQuery( }); const { sql: pfSQL, params: pfParams } = getPropertyFilterQuery(eventFilters, 'event', timezone); - const count = await rawQuery( - ` - select count() as num - from umami.event_data_pivot - any left join ( - select * - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = 2 - and event_name = {eventName:String}) website_event - on website_event.event_id = event_data_pivot.event_id - and website_event.session_id = event_data_pivot.session_id - and website_event.website_id = event_data_pivot.website_id - ${cohortQuery} - where event_data_pivot.website_id = {websiteId:UUID} - and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_data_pivot.event_name = {eventName:String} - ${filterQuery} - ${pfSQL} - `, - { ...queryParams, eventName, ...pfParams }, - ).then((res: any) => res[0].num); - - const data = await rawQuery( + return pagedRawQuery( ` select event_data_pivot.event_id as eventId, @@ -224,11 +148,9 @@ async function clickhouseQuery( event_data_pivot.url_path, event_data_pivot.created_at order by event_data_pivot.created_at desc - limit ${size} offset ${offset} `, { ...queryParams, eventName, ...pfParams }, + filters, FUNCTION_NAME, ); - - return { data, count, page: +page, pageSize: size }; } diff --git a/src/queries/sql/sessions/getSessionDataPivot.ts b/src/queries/sql/sessions/getSessionDataPivot.ts index ac4528123..c4e460b0e 100644 --- a/src/queries/sql/sessions/getSessionDataPivot.ts +++ b/src/queries/sql/sessions/getSessionDataPivot.ts @@ -1,5 +1,4 @@ import clickhouse from '@/lib/clickhouse'; -import { DEFAULT_PAGE_SIZE } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { PageResult, PropertyFilter, QueryFilters, SessionDataPivotRow } from '@/lib/types'; @@ -25,12 +24,9 @@ async function relationalQuery( propertyName: string, filters: QueryFilters, propertyFilters: PropertyFilter[] = [], -): Promise> { +) { const { timezone = 'utc' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma; - const { page = 1, pageSize } = filters; - const size = +pageSize || DEFAULT_PAGE_SIZE; - const offset = +size * (+page - 1); + const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma; const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, @@ -43,47 +39,7 @@ async function relationalQuery( timezone, ); - const countResult = (await rawQuery( - ` - with filtered_sessions as ( - select distinct website_event.session_id - from website_event - ${cohortQuery} - ${joinSessionQuery} - where website_event.website_id = {{websiteId::uuid}} - and website_event.created_at between {{startDate}} and {{endDate}} - ${filterQuery} - ${pfSQL} - ), - latest_session_properties as ( - select - ranked.session_id, - ranked.data_key - from ( - select - session_data.session_id, - session_data.data_key, - row_number() over ( - partition by session_data.session_id, session_data.data_key - order by session_data.created_at desc, session_data.session_data_id desc - ) as row_num - from session_data - join filtered_sessions - on filtered_sessions.session_id = session_data.session_id - where session_data.website_id = {{websiteId::uuid}} - ) ranked - where ranked.row_num = 1 - ) - select count(*) as num - from latest_session_properties - where latest_session_properties.data_key = {{propertyName}} - `, - { ...queryParams, websiteId, propertyName, ...pfParams }, - )) as { num: number }[]; - - const count = countResult[0].num; - - const rows = (await rawQuery( + return pagedRawQuery( ` with filtered_sessions as ( select distinct website_event.session_id @@ -132,8 +88,6 @@ async function relationalQuery( latest_session_properties.created_at as sort_created_at from latest_session_properties where latest_session_properties.data_key = {{propertyName}} - order by latest_session_properties.created_at desc - limit ${size} offset ${offset} ) select latest_session_properties.session_id as "sessionId", @@ -158,10 +112,9 @@ async function relationalQuery( order by paged_sessions.sort_created_at desc `, { ...queryParams, websiteId, propertyName, ...pfParams }, + filters, FUNCTION_NAME, - )) as SessionDataPivotRow[]; - - return { data: rows, count, page: +page, pageSize: size }; + ); } async function clickhouseQuery( @@ -169,12 +122,9 @@ async function clickhouseQuery( propertyName: string, filters: QueryFilters, propertyFilters: PropertyFilter[] = [], -): Promise> { +) { const { timezone = 'UTC' } = filters; - const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse; - const { page = 1, pageSize } = filters; - const size = +pageSize || DEFAULT_PAGE_SIZE; - const offset = +size * (+page - 1); + const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone }); const { sql: pfSQL, params: pfParams } = getPropertyFilterQuery( @@ -183,38 +133,7 @@ async function clickhouseQuery( timezone, ); - const countResult = (await rawQuery( - ` - with filtered_sessions as ( - select distinct website_event.session_id - from website_event - ${cohortQuery} - where website_event.website_id = {websiteId:UUID} - and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64} - ${filterQuery} - ${pfSQL} - ), - latest_session_properties as ( - select - session_data.session_id as session_id, - session_data.data_key as data_key - from session_data final - join filtered_sessions - on filtered_sessions.session_id = session_data.session_id - where session_data.website_id = {websiteId:UUID} - group by - session_data.session_id, - session_data.data_key - ) - select count() as num - from latest_session_properties - where latest_session_properties.data_key = {propertyName:String} - `, - { ...queryParams, websiteId, propertyName, ...pfParams }, - )) as { num: number }[]; - const count = countResult[0].num; - - const data = (await rawQuery( + return pagedRawQuery( ` with filtered_sessions as ( select distinct website_event.session_id @@ -249,8 +168,6 @@ async function clickhouseQuery( latest_session_properties.created_at as sort_created_at from latest_session_properties where latest_session_properties.data_key = {propertyName:String} - order by latest_session_properties.created_at desc - limit ${size} offset ${offset} ) select latest_session_properties.session_id as sessionId, @@ -274,8 +191,7 @@ async function clickhouseQuery( order by paged_sessions.sort_created_at desc `, { ...queryParams, websiteId, propertyName, ...pfParams }, + filters, FUNCTION_NAME, - )) as SessionDataPivotRow[]; - - return { data, count, page: +page, pageSize: size }; + ); }