diff --git a/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx b/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx index b4c455cb0..95ca3e883 100644 --- a/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx +++ b/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx @@ -1,7 +1,9 @@ 'use client'; import { Column } from '@umami/react-zen'; import { useEffect, useRef, useState } from 'react'; +import { Empty } from '@/components/common/Empty'; import { useMobile } from '@/components/hooks'; +import { hasReplayFullSnapshot } from '@/lib/replay'; import 'rrweb-player/dist/style.css'; export function ReplayPlayer({ events }: { events: any[] }) { @@ -9,12 +11,14 @@ export function ReplayPlayer({ events }: { events: any[] }) { const playerRef = useRef(null); const [loaded, setLoaded] = useState(false); const { isMobile, isPhone } = useMobile(); + const canReplay = hasReplayFullSnapshot(events); + const showUnavailable = !events?.length || !canReplay; const playerWidth = isPhone ? 360 : isMobile ? 640 : 1024; const playerHeight = isPhone ? 202 : isMobile ? 360 : 576; useEffect(() => { - if (!containerRef.current || !events?.length) return; + if (!containerRef.current || !events?.length || !canReplay) return; import('rrweb-player').then(mod => { const RRWebPlayer = mod.default; @@ -46,7 +50,7 @@ export function ReplayPlayer({ events }: { events: any[] }) { playerRef.current = null; } }; - }, [events, playerWidth, playerHeight]); + }, [canReplay, events, playerWidth, playerHeight]); return ( @@ -54,14 +58,16 @@ export function ReplayPlayer({ events }: { events: any[] }) { ref={containerRef} style={{ width: playerWidth, - minHeight: loaded ? undefined : playerHeight, + minHeight: loaded && canReplay ? undefined : playerHeight, maxWidth: '100%', overflow: 'hidden', borderRadius: '8px', border: '1px solid var(--base300)', background: 'var(--base75)', }} - /> + > + {showUnavailable && } + ); } diff --git a/src/app/api/record/route.ts b/src/app/api/record/route.ts index 7fa267b79..ade6aa0c2 100644 --- a/src/app/api/record/route.ts +++ b/src/app/api/record/route.ts @@ -7,6 +7,7 @@ import { getClientInfo, hasBlockedIp } from '@/lib/detect'; import { parseToken } from '@/lib/jwt'; import { fetchAccount, fetchTeam } from '@/lib/load'; import { getRecorderConfig } from '@/lib/recorder'; +import { getReplayEventCount } from '@/lib/replay'; import { parseRequest } from '@/lib/request'; import { badRequest, forbidden, json, payloadTooLarge, serverError } from '@/lib/response'; import { getWebsite } from '@/queries/prisma'; @@ -197,7 +198,7 @@ export async function POST(request: Request) { visitId, chunkIndex, events, - eventCount: events.length, + eventCount: getReplayEventCount(events), startedAt, endedAt, }); diff --git a/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts b/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts index a35afae55..1720a4501 100644 --- a/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts +++ b/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts @@ -1,3 +1,4 @@ +import { restoreReplayEventFragments } from '@/lib/replay'; import { parseRequest } from '@/lib/request'; import { json, unauthorized } from '@/lib/response'; import { canViewAuthenticatedWebsite } from '@/permissions'; @@ -91,7 +92,9 @@ export async function GET( } const chunks = await getReplayChunks(websiteId, replayId, { endAt, endChunkIndex }); - const allEvents = mergeReplayEvents(chunks, { until, endChunkIndex, endEventIndex }); + const allEvents = restoreReplayEventFragments( + mergeReplayEvents(chunks, { until, endChunkIndex, endEventIndex }), + ); const sessionId = chunks.length > 0 ? chunks[0].sessionId : null; const startedAt = chunks.length > 0 ? chunks[0].startedAt : null; const endedAt = chunks.length > 0 ? chunks[chunks.length - 1].endedAt : null; diff --git a/src/lib/replay.test.ts b/src/lib/replay.test.ts new file mode 100644 index 000000000..d875b10e5 --- /dev/null +++ b/src/lib/replay.test.ts @@ -0,0 +1,84 @@ +import { expect, test } from 'vitest'; +import { + getReplayEventCount, + hasReplayFullSnapshot, + REPLAY_EVENT_FRAGMENT_TYPE, + restoreReplayEventFragments, +} from './replay'; + +test('hasReplayFullSnapshot returns true when a full snapshot is present', () => { + expect(hasReplayFullSnapshot([{ type: 4 }, { type: 2 }, { type: 3 }])).toBe(true); +}); + +test('hasReplayFullSnapshot returns false when only incremental events are present', () => { + expect(hasReplayFullSnapshot([{ type: 4 }, { type: 3 }, { type: 3 }])).toBe(false); +}); + +test('hasReplayFullSnapshot handles missing events', () => { + expect(hasReplayFullSnapshot(null)).toBe(false); + expect(hasReplayFullSnapshot(undefined)).toBe(false); +}); + +test('restoreReplayEventFragments restores fragmented events', () => { + const fullSnapshot = { + type: 2, + timestamp: 1781553116151, + data: { + node: { + type: 0, + childNodes: [{ type: 2, tagName: 'html' }], + }, + }, + }; + const serialized = JSON.stringify(fullSnapshot); + const splitAt = Math.floor(serialized.length / 2); + + const events = restoreReplayEventFragments([ + { type: 4, timestamp: 1781553116150 }, + { + type: REPLAY_EVENT_FRAGMENT_TYPE, + timestamp: fullSnapshot.timestamp, + data: { + id: 'snapshot-1', + index: 0, + total: 2, + value: serialized.slice(0, splitAt), + }, + }, + { + type: REPLAY_EVENT_FRAGMENT_TYPE, + timestamp: fullSnapshot.timestamp, + data: { + id: 'snapshot-1', + index: 1, + total: 2, + value: serialized.slice(splitAt), + }, + }, + { type: 3, timestamp: 1781553116160 }, + ]); + + expect(events).toEqual([ + { type: 4, timestamp: 1781553116150 }, + fullSnapshot, + { type: 3, timestamp: 1781553116160 }, + ]); + expect(hasReplayFullSnapshot(events)).toBe(true); +}); + +test('getReplayEventCount counts a fragment group as one event', () => { + expect( + getReplayEventCount([ + { type: 4 }, + { + type: REPLAY_EVENT_FRAGMENT_TYPE, + data: { id: 'snapshot-1', index: 0, total: 2, value: '{' }, + }, + { + type: REPLAY_EVENT_FRAGMENT_TYPE, + data: { id: 'snapshot-1', index: 1, total: 2, value: '}' }, + }, + { type: 3 }, + ]), + ).toBe(3); +}); diff --git a/src/lib/replay.ts b/src/lib/replay.ts new file mode 100644 index 000000000..0ad8c96ba --- /dev/null +++ b/src/lib/replay.ts @@ -0,0 +1,108 @@ +export const RRWEB_EVENT_TYPE = { + FullSnapshot: 2, +} as const; + +export const REPLAY_EVENT_FRAGMENT_TYPE = 'umami:rrweb-event-fragment'; + +interface ReplayEventFragment { + type: typeof REPLAY_EVENT_FRAGMENT_TYPE; + timestamp?: number; + data: { + id: string; + index: number; + total: number; + value: string; + }; +} + +export function isReplayEventFragment(event: any): event is ReplayEventFragment { + const { data } = event || {}; + + return ( + event?.type === REPLAY_EVENT_FRAGMENT_TYPE && + typeof data?.id === 'string' && + Number.isInteger(data?.index) && + Number.isInteger(data?.total) && + data.index >= 0 && + data.index < data.total && + typeof data?.value === 'string' + ); +} + +export function getReplayEventCount(events: any[] | null | undefined) { + if (!Array.isArray(events)) { + return 0; + } + + return events.reduce((count, event) => { + if (isReplayEventFragment(event)) { + return count + (event.data.index === 0 ? 1 : 0); + } + + return count + 1; + }, 0); +} + +export function restoreReplayEventFragments(events: any[] | null | undefined) { + if (!Array.isArray(events)) { + return []; + } + + const restored: any[] = []; + const pending = new Map< + string, + { + total: number; + values: Map; + received: number; + } + >(); + + for (const event of events) { + if (!isReplayEventFragment(event)) { + restored.push(event); + continue; + } + + const { id, index, total, value } = event.data; + let fragment = pending.get(id); + + if (!fragment || fragment.total !== total) { + fragment = { + total, + values: new Map(), + received: 0, + }; + pending.set(id, fragment); + } + + if (!fragment.values.has(index)) { + fragment.values.set(index, value); + fragment.received += 1; + } + + if (fragment.received === fragment.total) { + pending.delete(id); + + try { + let serialized = ''; + + for (let i = 0; i < fragment.total; i++) { + serialized += fragment.values.get(i) || ''; + } + + restored.push(JSON.parse(serialized)); + } catch { + // Ignore malformed fragment groups. A partial replay is better than failing the whole response. + } + } + } + + return restored; +} + +export function hasReplayFullSnapshot(events: any[] | null | undefined) { + return ( + Array.isArray(events) && events.some(event => event?.type === RRWEB_EVENT_TYPE.FullSnapshot) + ); +} diff --git a/src/recorder/index.js b/src/recorder/index.js index 9f5950ec3..1d36341cb 100644 --- a/src/recorder/index.js +++ b/src/recorder/index.js @@ -23,7 +23,9 @@ import { record } from 'rrweb'; const REPLAY_FLUSH_EVENT_COUNT = 100; const REPLAY_FLUSH_INTERVAL = 2000; - const REPLAY_MAX_PAYLOAD_SIZE = 900000; + const REPLAY_MAX_PAYLOAD_SIZE = 950000; + const REPLAY_FRAGMENT_TYPE = 'umami:rrweb-event-fragment'; + const REPLAY_FRAGMENT_TOTAL_PLACEHOLDER = 999999999; const HEATMAP_FLUSH_EVENT_COUNT = 20; const HEATMAP_FLUSH_INTERVAL = 5000; @@ -41,6 +43,7 @@ import { record } from 'rrweb'; let replayFlushTimer = null; let heatmapFlushTimer = null; let replayStartTime = null; + let replayLastChunkIndex = 0; let replayStopped = false; let heatmapStarted = false; @@ -69,6 +72,35 @@ import { record } from 'rrweb'; const isReplayPayloadTooLarge = (events, timestamp) => getReplayPayloadSize(events, timestamp) > REPLAY_MAX_PAYLOAD_SIZE; + const getReplayChunkIndex = () => { + const timestamp = Math.floor(Date.now() / 1000); + const chunkIndex = Math.max(timestamp, replayLastChunkIndex + 1); + + replayLastChunkIndex = chunkIndex; + + return chunkIndex; + }; + + const createReplayFragment = (id, index, total, timestamp, value) => ({ + type: REPLAY_FRAGMENT_TYPE, + timestamp, + data: { + id, + index, + total, + value, + }, + }); + + const getReplayEventTimestamp = event => { + const timestamp = Number(event?.timestamp); + + return Number.isFinite(timestamp) ? timestamp : Date.now(); + }; + + const getReplayFragmentId = () => + `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + const sendPayload = (type, payload, useKeepalive = false) => { const cache = getSessionCache(); @@ -90,45 +122,106 @@ import { record } from 'rrweb'; }).catch(() => {}); }; + const sendReplayChunk = (events, timestamp, useKeepalive = false) => { + replayLastChunkIndex = Math.max(replayLastChunkIndex, timestamp); + + sendPayload( + 'record', + { + events, + timestamp, + }, + useKeepalive, + ); + }; + + const getReplayEventFragments = (event, chunkTimestamp) => { + const value = JSON.stringify(event); + const id = getReplayFragmentId(); + const eventTimestamp = getReplayEventTimestamp(event); + const fragments = []; + let start = 0; + + while (start < value.length) { + let low = start + 1; + let high = value.length; + let end = start; + + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const fragmentChunkTimestamp = chunkTimestamp + fragments.length; + const fragment = createReplayFragment( + id, + fragments.length, + REPLAY_FRAGMENT_TOTAL_PLACEHOLDER, + eventTimestamp, + value.slice(start, mid), + ); + + if (isReplayPayloadTooLarge([fragment], fragmentChunkTimestamp)) { + high = mid - 1; + } else { + end = mid; + low = mid + 1; + } + } + + if (end === start) { + end = start + 1; + } + + fragments.push(value.slice(start, end)); + start = end; + } + + return fragments.map((fragment, index) => + createReplayFragment(id, index, fragments.length, eventTimestamp, fragment), + ); + }; + + const sendReplayEventFragments = (event, timestamp, useKeepalive = false) => { + const fragments = getReplayEventFragments(event, timestamp); + + fragments.forEach((fragment, index) => { + sendReplayChunk([fragment], timestamp + index, useKeepalive); + }); + + return fragments.length; + }; + const sendReplayEvents = (events, timestamp, useKeepalive = false) => { let chunk = []; let chunkOffset = 0; - events.forEach(event => { - const candidate = [...chunk, event]; + for (const event of events) { + const chunkTimestamp = timestamp + chunkOffset; - if (isReplayPayloadTooLarge(candidate, timestamp + chunkOffset)) { + if (isReplayPayloadTooLarge([event], chunkTimestamp)) { if (chunk.length) { - sendPayload( - 'record', - { - events: chunk, - timestamp: timestamp + chunkOffset, - }, - useKeepalive, - ); + sendReplayChunk(chunk, chunkTimestamp, useKeepalive); chunk = []; chunkOffset += 1; } - if (isReplayPayloadTooLarge([event], timestamp + chunkOffset)) { + chunkOffset += sendReplayEventFragments(event, timestamp + chunkOffset, useKeepalive); + continue; + } + + const candidate = [...chunk, event]; + + if (isReplayPayloadTooLarge(candidate, chunkTimestamp)) { + if (chunk.length) { + sendReplayChunk(chunk, chunkTimestamp, useKeepalive); + chunk = []; chunkOffset += 1; - return; } } chunk.push(event); - }); + } if (chunk.length) { - sendPayload( - 'record', - { - events: chunk, - timestamp: timestamp + chunkOffset, - }, - useKeepalive, - ); + sendReplayChunk(chunk, timestamp + chunkOffset, useKeepalive); } }; @@ -138,7 +231,7 @@ import { record } from 'rrweb'; const events = replayBuffer; replayBuffer = []; - sendReplayEvents(events, Math.floor(Date.now() / 1000), useKeepalive); + sendReplayEvents(events, getReplayChunkIndex(), useKeepalive); }; const flushHeatmap = (useKeepalive = false) => { @@ -326,10 +419,21 @@ import { record } from 'rrweb'; return; } - if ( - replayBuffer.length && - isReplayPayloadTooLarge([...replayBuffer, event], Math.floor(Date.now() / 1000)) - ) { + const timestamp = Math.floor(Date.now() / 1000); + + if (isReplayPayloadTooLarge([event], timestamp)) { + if (replayBuffer.length) { + const events = replayBuffer; + replayBuffer = []; + + sendReplayEvents(events, getReplayChunkIndex()); + } + + sendReplayEvents([event], getReplayChunkIndex()); + return; + } + + if (replayBuffer.length && isReplayPayloadTooLarge([...replayBuffer, event], timestamp)) { flushReplay(); } @@ -359,6 +463,11 @@ import { record } from 'rrweb'; checkoutEveryNms: 30000, ...(blockSelector && { blockSelector }), }); + + if (replayStopped && replayStopFn) { + replayStopFn(); + replayStopFn = null; + } }; const beginHeatmapCapture = () => {