import clickhouse from '@/lib/clickhouse'; import { HEATMAP_EVENT_TYPE } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; const FUNCTION_NAME = 'getHeatmap'; const POINT_LIMIT = 5000; const PAGE_LIMIT = 100; const SCROLL_BUCKET_SIZE = 5; export type HeatmapMode = 'click' | 'scroll'; export interface HeatmapParameters extends QueryFilters { urlPath?: string; mode?: HeatmapMode; } export interface HeatmapPage { urlPath: string; count: number; sessions: number; } export interface HeatmapPoint { nodeId: number | null; x: number; y: number; viewportW: number; viewportH: number; count: number; } export interface HeatmapScrollBucket { depth: number; sessions: number; } export interface HeatmapSnapshot { replayId: string; timestamp: number; chunkIndex: number | null; eventIndex: number | null; } export interface HeatmapResult { mode: HeatmapMode; pages: HeatmapPage[]; points: HeatmapPoint[]; snapshot: HeatmapSnapshot | null; scroll: { buckets: HeatmapScrollBucket[]; totalSessions: number; pageH: number | null; viewportW: number | null; viewportH: number | null; }; } export async function getHeatmap( websiteId: string, parameters: HeatmapParameters, ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(websiteId, parameters), [CLICKHOUSE]: () => clickhouseQuery(websiteId, parameters), }); } interface SnapshotRow { replayId: string; timestamp: number | string; chunkIndex: number | string | null; eventIndex: number | string | null; } interface HeatmapFilterContext { joinQuery: string; queryParams: Record; } async function relationalQuery( websiteId: string, parameters: HeatmapParameters, ): Promise { const { rawQuery } = prisma; const { startDate, endDate, urlPath, mode = 'click' } = parameters; const eventType = mode === 'scroll' ? HEATMAP_EVENT_TYPE.scroll : HEATMAP_EVENT_TYPE.click; const filterContext = getRelationalHeatmapFilterContext(websiteId, parameters); const clickPageFilter = mode === 'click' ? ` and x is not null and y is not null and viewport_w is not null and viewport_h is not null ` : ''; const pages: HeatmapPage[] = await rawQuery( ` select h.url_path as "urlPath", count(*)::int as count, count(distinct h.visit_id)::int as sessions from heatmap_event h ${filterContext.joinQuery} where h.website_id = {{websiteId::uuid}} and h.event_type = {{eventType}} and h.created_at between {{startDate}} and {{endDate}} ${clickPageFilter} group by h.url_path order by sessions desc, count desc limit ${PAGE_LIMIT} `, { ...filterContext.queryParams, websiteId, eventType, startDate, endDate }, FUNCTION_NAME, ); if (!urlPath) { return { mode, pages, points: [], snapshot: null, scroll: emptyScroll() }; } if (mode === 'scroll') { const bucketRows: { depth: number | string; sessions: number | string }[] = await rawQuery( ` select (floor(max_pct / ${SCROLL_BUCKET_SIZE}) * ${SCROLL_BUCKET_SIZE})::int as depth, count(*)::int as sessions from ( select h.visit_id, max(h.scroll_pct) as max_pct from heatmap_event h ${filterContext.joinQuery} where h.website_id = {{websiteId::uuid}} and h.event_type = {{eventType}} and h.url_path = {{urlPath}} and h.created_at between {{startDate}} and {{endDate}} and h.scroll_pct is not null group by h.visit_id ) per_session group by depth order by depth `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate }, FUNCTION_NAME, ); const dimRows: { totalSessions: number | string; pageH: number | null; viewportW: number | null; viewportH: number | null; }[] = await rawQuery( ` select count(distinct h.visit_id)::int as "totalSessions", (mode() within group (order by h.page_h))::int as "pageH", (mode() within group (order by h.viewport_w))::int as "viewportW", (mode() within group (order by h.viewport_h))::int as "viewportH" from heatmap_event h ${filterContext.joinQuery} where h.website_id = {{websiteId::uuid}} and h.event_type = {{eventType}} and h.url_path = {{urlPath}} and h.created_at between {{startDate}} and {{endDate}} and h.scroll_pct is not null `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate }, FUNCTION_NAME, ); const dim = dimRows[0]; const scroll = { buckets: bucketRows.map(r => ({ depth: Number(r.depth), sessions: Number(r.sessions) })), totalSessions: Number(dim?.totalSessions ?? 0), pageH: dim?.pageH ?? null, viewportW: dim?.viewportW ?? null, viewportH: dim?.viewportH ?? null, }; const snapshot = await getRelationalSnapshot(rawQuery, { websiteId, eventType, urlPath, startDate, endDate, viewportW: scroll.viewportW, viewportH: scroll.viewportH, filterContext, }); return { mode, pages, points: [], snapshot, scroll, }; } const rawPoints: HeatmapPoint[] = await rawQuery( ` select h.node_id as "nodeId", h.x, h.y, h.viewport_w as "viewportW", h.viewport_h as "viewportH", count(*)::int as count from heatmap_event h ${filterContext.joinQuery} where h.website_id = {{websiteId::uuid}} and h.event_type = {{eventType}} and h.url_path = {{urlPath}} and h.created_at between {{startDate}} and {{endDate}} and h.x is not null and h.y is not null and h.viewport_w is not null and h.viewport_h is not null group by h.node_id, h.x, h.y, h.viewport_w, h.viewport_h order by count desc limit ${POINT_LIMIT} `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate }, FUNCTION_NAME, ); const viewport = pickSnapshotViewport(rawPoints); const snapshot = await getRelationalSnapshot(rawQuery, { websiteId, eventType, urlPath, startDate, endDate, viewportW: viewport?.width ?? null, viewportH: viewport?.height ?? null, filterContext, }); return { mode, pages, points: rawPoints, snapshot, scroll: emptyScroll() }; } async function getRelationalSnapshot( rawQuery: typeof prisma.rawQuery, { websiteId, eventType, urlPath, startDate, endDate, viewportW, viewportH, filterContext, }: { websiteId: string; eventType: number; urlPath: string; startDate: Date; endDate: Date; viewportW: number | null; viewportH: number | null; filterContext: HeatmapFilterContext; }, ): Promise { const viewportFilter = viewportW && viewportH ? ` and h.viewport_w = {{viewportW}} and h.viewport_h = {{viewportH}} ` : ''; const rows: SnapshotRow[] = await rawQuery( ` with best_visit as ( select h.visit_id as visit_id, count(*) as event_count, min(h.created_at) as first_seen from heatmap_event h ${filterContext.joinQuery} where h.website_id = {{websiteId::uuid}} and h.event_type = {{eventType}} and h.url_path = {{urlPath}} and h.created_at between {{startDate}} and {{endDate}} ${viewportFilter} group by h.visit_id order by event_count desc, first_seen asc limit 1 ) select h.visit_id as "replayId", coalesce(h.replay_time_ms, (extract(epoch from h.created_at) * 1000)::bigint) as "timestamp", h.replay_chunk_index as "chunkIndex", h.replay_event_index as "eventIndex" from heatmap_event h inner join best_visit bv on bv.visit_id = h.visit_id inner join ( select distinct visit_id from session_replay where website_id = {{websiteId::uuid}} ) sr on sr.visit_id = h.visit_id ${filterContext.joinQuery} where h.website_id = {{websiteId::uuid}} and h.event_type = {{eventType}} and h.url_path = {{urlPath}} and h.created_at between {{startDate}} and {{endDate}} ${viewportFilter} order by case when h.replay_chunk_index is null then 1 else 0 end asc, h.replay_chunk_index asc nulls last, h.replay_event_index asc nulls last, h.created_at asc limit 1 `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate, viewportW, viewportH }, FUNCTION_NAME, ); return mapSnapshot(rows[0]); } async function clickhouseQuery( websiteId: string, parameters: HeatmapParameters, ): Promise { const { rawQuery } = clickhouse; const { startDate, endDate, urlPath, mode = 'click' } = parameters; const eventType = mode === 'scroll' ? HEATMAP_EVENT_TYPE.scroll : HEATMAP_EVENT_TYPE.click; const filterContext = getClickhouseHeatmapFilterContext(websiteId, parameters); const clickPageFilter = mode === 'click' ? ` and x is not null and y is not null and viewport_w is not null and viewport_h is not null ` : ''; const pageRows = await rawQuery< { urlPath: string; count: string | number; sessions: string | number }[] >( ` select h.url_path as urlPath, count() as count, uniq(h.visit_id) as sessions from heatmap_event h ${filterContext.joinQuery} where h.website_id = {websiteId:UUID} and h.event_type = {eventType:UInt8} and h.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${clickPageFilter} group by h.url_path order by sessions desc, count desc limit ${PAGE_LIMIT} `, { ...filterContext.queryParams, websiteId, eventType, startDate, endDate }, FUNCTION_NAME, ); const pages: HeatmapPage[] = pageRows.map(p => ({ urlPath: p.urlPath, count: Number(p.count), sessions: Number(p.sessions), })); if (!urlPath) { return { mode, pages, points: [], snapshot: null, scroll: emptyScroll() }; } if (mode === 'scroll') { const bucketRows = await rawQuery<{ depth: number | string; sessions: number | string }[]>( ` select intDiv(max_pct, ${SCROLL_BUCKET_SIZE}) * ${SCROLL_BUCKET_SIZE} as depth, count() as sessions from ( select h.visit_id, max(h.scroll_pct) as max_pct from heatmap_event h ${filterContext.joinQuery} where h.website_id = {websiteId:UUID} and h.event_type = {eventType:UInt8} and h.url_path = {urlPath:String} and h.created_at between {startDate:DateTime64} and {endDate:DateTime64} and h.scroll_pct is not null group by h.visit_id ) group by depth order by depth `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate }, FUNCTION_NAME, ); const dimRows = await rawQuery< { totalSessions: number | string; pageH: number | null; viewportW: number | null; viewportH: number | null; }[] >( ` select uniq(h.visit_id) as totalSessions, toInt32OrNull(toString(arrayElement(topK(1)(h.page_h), 1))) as pageH, toInt32OrNull(toString(arrayElement(topK(1)(h.viewport_w), 1))) as viewportW, toInt32OrNull(toString(arrayElement(topK(1)(h.viewport_h), 1))) as viewportH from heatmap_event h ${filterContext.joinQuery} where h.website_id = {websiteId:UUID} and h.event_type = {eventType:UInt8} and h.url_path = {urlPath:String} and h.created_at between {startDate:DateTime64} and {endDate:DateTime64} and h.scroll_pct is not null `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate }, FUNCTION_NAME, ); const dim = dimRows[0]; const scroll = { buckets: bucketRows.map(r => ({ depth: Number(r.depth), sessions: Number(r.sessions) })), totalSessions: Number(dim?.totalSessions ?? 0), pageH: dim?.pageH === null || dim?.pageH === undefined ? null : Number(dim.pageH), viewportW: dim?.viewportW === null || dim?.viewportW === undefined ? null : Number(dim.viewportW), viewportH: dim?.viewportH === null || dim?.viewportH === undefined ? null : Number(dim.viewportH), }; const snapshot = await getClickhouseSnapshot(rawQuery, { websiteId, eventType, urlPath, startDate, endDate, viewportW: scroll.viewportW, viewportH: scroll.viewportH, filterContext, }); return { mode, pages, points: [], snapshot, scroll, }; } const pointRows = await rawQuery< { nodeId: number | null; x: number; y: number; viewportW: number; viewportH: number; count: string | number; }[] >( ` select h.node_id as nodeId, h.x, h.y, h.viewport_w as viewportW, h.viewport_h as viewportH, count() as count from heatmap_event h ${filterContext.joinQuery} where h.website_id = {websiteId:UUID} and h.event_type = {eventType:UInt8} and h.url_path = {urlPath:String} and h.created_at between {startDate:DateTime64} and {endDate:DateTime64} and h.x is not null and h.y is not null and h.viewport_w is not null and h.viewport_h is not null group by h.node_id, h.x, h.y, h.viewport_w, h.viewport_h order by count desc limit ${POINT_LIMIT} `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate }, FUNCTION_NAME, ); const points: HeatmapPoint[] = pointRows.map(p => ({ nodeId: p.nodeId === null || p.nodeId === undefined ? null : Number(p.nodeId), x: Number(p.x), y: Number(p.y), viewportW: Number(p.viewportW), viewportH: Number(p.viewportH), count: Number(p.count), })); const viewport = pickSnapshotViewport(points); const snapshot = await getClickhouseSnapshot(rawQuery, { websiteId, eventType, urlPath, startDate, endDate, viewportW: viewport?.width ?? null, viewportH: viewport?.height ?? null, filterContext, }); return { mode, pages, points, snapshot, scroll: emptyScroll() }; } async function getClickhouseSnapshot( rawQuery: typeof clickhouse.rawQuery, { websiteId, eventType, urlPath, startDate, endDate, viewportW, viewportH, filterContext, }: { websiteId: string; eventType: number; urlPath: string; startDate: Date; endDate: Date; viewportW: number | null; viewportH: number | null; filterContext: HeatmapFilterContext; }, ): Promise { const viewportFilter = viewportW && viewportH ? ` and h.viewport_w = {viewportW:UInt32} and h.viewport_h = {viewportH:UInt32} ` : ''; const rows = await rawQuery( ` with best_visit as ( select h.visit_id as visit_id, count() as event_count, min(h.created_at) as first_seen from heatmap_event h ${filterContext.joinQuery} where h.website_id = {websiteId:UUID} and h.event_type = {eventType:UInt8} and h.url_path = {urlPath:String} and h.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${viewportFilter} group by h.visit_id order by event_count desc, first_seen asc limit 1 ) select toString(h.visit_id) as replayId, ifNull(h.replay_time_ms, toInt64(toUnixTimestamp(h.created_at)) * 1000) as timestamp, h.replay_chunk_index as chunkIndex, h.replay_event_index as eventIndex from heatmap_event h inner join best_visit bv on bv.visit_id = h.visit_id inner join ( select distinct visit_id from session_replay where website_id = {websiteId:UUID} ) sr on sr.visit_id = h.visit_id ${filterContext.joinQuery} where h.website_id = {websiteId:UUID} and h.event_type = {eventType:UInt8} and h.url_path = {urlPath:String} and h.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${viewportFilter} order by isNull(h.replay_chunk_index) asc, h.replay_chunk_index asc, h.replay_event_index asc, h.created_at asc limit 1 `, { ...filterContext.queryParams, websiteId, eventType, urlPath, startDate, endDate, viewportW, viewportH, }, FUNCTION_NAME, ); return mapSnapshot(rows[0]); } function emptyScroll(): HeatmapResult['scroll'] { return { buckets: [], totalSessions: 0, pageH: null, viewportW: null, viewportH: null, }; } function pickSnapshotViewport(points: HeatmapPoint[]): { width: number; height: number } | null { const buckets = new Map(); for (const p of points) { const key = `${p.viewportW}x${p.viewportH}`; const existing = buckets.get(key); if (existing) { existing.count += p.count; } else { buckets.set(key, { width: p.viewportW, height: p.viewportH, count: p.count }); } } let best: { width: number; height: number; count: number } | null = null; for (const bucket of buckets.values()) { if (!best || bucket.count > best.count) { best = bucket; } } return best ? { width: best.width, height: best.height } : null; } function mapSnapshot(row?: SnapshotRow | null): HeatmapSnapshot | null { if (!row) { return null; } return { replayId: row.replayId, timestamp: Number(row.timestamp), chunkIndex: row.chunkIndex === null || row.chunkIndex === undefined ? null : Number(row.chunkIndex), eventIndex: row.eventIndex === null || row.eventIndex === undefined ? null : Number(row.eventIndex), }; } function getRelationalHeatmapFilterContext( websiteId: string, filters: QueryFilters, ): HeatmapFilterContext { const { parseFilters } = prisma; const { filterQuery, cohortQuery, excludeBounceQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, }); if (!(filterQuery || cohortQuery || excludeBounceQuery)) { return { joinQuery: '', queryParams }; } return { joinQuery: ` inner join ( select distinct website_event.website_id, website_event.session_id, website_event.visit_id from website_event ${joinSessionQuery} ${cohortQuery} ${excludeBounceQuery} where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} ${filterQuery} ) filtered_visits on filtered_visits.website_id = h.website_id and filtered_visits.session_id = h.session_id and filtered_visits.visit_id = h.visit_id `, queryParams, }; } function getClickhouseHeatmapFilterContext( websiteId: string, filters: QueryFilters, ): HeatmapFilterContext { const { parseFilters } = clickhouse; const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({ ...filters, websiteId, }); if (!(filterQuery || cohortQuery || excludeBounceQuery)) { return { joinQuery: '', queryParams }; } return { joinQuery: ` inner join ( select distinct website_id, session_id, visit_id from website_event ${cohortQuery} ${excludeBounceQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} ) filtered_visits on filtered_visits.website_id = h.website_id and filtered_visits.session_id = h.session_id and filtered_visits.visit_id = h.visit_id `, queryParams, }; }