diff --git a/package.json b/package.json index 9a71fddd3..812040b5a 100644 --- a/package.json +++ b/package.json @@ -112,8 +112,8 @@ "react-window": "^2.2.7", "redis": "^5.12.1", "request-ip": "^3.3.0", - "rrweb": "2.0.0-alpha.4", - "rrweb-player": "1.0.0-alpha.4", + "rrweb": "^2.0.1", + "rrweb-player": "^2.0.1", "semver": "^7.8.0", "serialize-error": "^13.0.1", "thenby": "^1.4.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e41918f28..3549d8907 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -186,11 +186,11 @@ importers: specifier: ^3.3.0 version: 3.3.0 rrweb: - specifier: 2.0.0-alpha.4 - version: 2.0.0-alpha.4 + specifier: ^2.0.1 + version: 2.0.1 rrweb-player: - specifier: 1.0.0-alpha.4 - version: 1.0.0-alpha.4 + specifier: ^2.0.1 + version: 2.0.1 semver: specifier: ^7.8.0 version: 7.8.0 @@ -2633,8 +2633,17 @@ packages: cpu: [x64] os: [win32] - '@rrweb/types@2.0.0-alpha.20': - resolution: {integrity: sha512-RbnDgKxA/odwB1R4gF7eUUj+rdSrq6ROQJsnMw7MIsGzlbSYvJeZN8YY4XqU0G6sKJvXI6bSzk7w/G94jNwzhw==} + '@rrweb/packer@2.0.1': + resolution: {integrity: sha512-2U8RkVe+ZK3gTtmKjJKjoh6QL1Q6wIlt3sBTJeCPERLr9e0JIimNrZ1+bog1L30CTVP3ULuDf5Ba7RZO+QNcHg==} + + '@rrweb/replay@2.0.1': + resolution: {integrity: sha512-UrZONirb8o4t0urg1aJ8aMSWhNHvJ3a2usuVbPJIm/edyFbwTvvY9sg/rrXccyXOntFhhp73R5nCretODRZcCw==} + + '@rrweb/types@2.0.1': + resolution: {integrity: sha512-x8cNaKFxdvS0/5IfKOj1xG8Htnw1fUwqoPqgP61bOaWqZL6t9UtRPpO9ZlMB2z+FiTxg9zlFFL+Da4ZpFuhoiA==} + + '@rrweb/utils@2.0.1': + resolution: {integrity: sha512-t+b+4rB23/e4EpoGMdPUr6cKtDsFuOb3LxA9IHE8e3A/Xo0nMUfH0bkwqdpUNa/VGDOdiGeG3bhHjuO9zWX54g==} '@schummar/icu-type-parser@1.21.5': resolution: {integrity: sha512-bXHSaW5jRTmke9Vd0h5P7BtWZG9Znqb8gSDxZnxaGSJnGwPLDPfS+3g0BKzeWqzgZPsIVZkM7m2tbo18cm5HBw==} @@ -5658,17 +5667,17 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true - rrdom@0.1.7: - resolution: {integrity: sha512-ZLd8f14z9pUy2Hk9y636cNv5Y2BMnNEY99wxzW9tD2BLDfe1xFxtLjB4q/xCBYo6HRe0wofzKzjm4JojmpBfFw==} + rrdom@2.0.1: + resolution: {integrity: sha512-z8GdQWFmVfa8GpcGkAqfBXseb89z0ciTPSBkID0qr0i6KHCye1MdQ4VoXFf0Ejp3ELe/c7AObu4zsbSbMT+hiw==} - rrweb-player@1.0.0-alpha.4: - resolution: {integrity: sha512-Wlmn9GZ5Fdqa37vd3TzsYdLl/JWEvXNUrLCrYpnOwEgmY409HwVIvvA5aIo7k582LoKgdRCsB87N+f0oWAR0Kg==} + rrweb-player@2.0.1: + resolution: {integrity: sha512-v8PVnOy7Q3kXqWriGuzZo4ndGEV06nCMRkH/jSHweJfTLwpLqb2evgMj0S5FwQLsN2X1VM+L2/i7QWGJfgj9zw==} - rrweb-snapshot@2.0.0-alpha.4: - resolution: {integrity: sha512-KQ2OtPpXO5jLYqg1OnXS/Hf+EzqnZyP5A+XPqBCjYpj3XIje/Od4gdUwjbFo3cVuWq5Cw5Y1d3/xwgIS7/XpQQ==} + rrweb-snapshot@2.0.1: + resolution: {integrity: sha512-H8TA29ct2tpYtr5oaB0N+ksRY9bhEErfAYcKbce4+w8lqEUQChpKZd41HpBJLkp04b5QUfWUg4v6r9Okf8r7vA==} - rrweb@2.0.0-alpha.4: - resolution: {integrity: sha512-wEHUILbxDPcNwkM3m4qgPgXAiBJyqCbbOHyVoNEVBJzHszWEFYyTbrZqUdeb1EfmTRC2PsumCIkVcomJ/xcOzA==} + rrweb@2.0.1: + resolution: {integrity: sha512-MQYzOtI7aZft9ffDMT6rmRSAj6hO4W6T7iA4eB9Mtcuiv3M7DbnnxDMrcPW0sQXepgYN46YhfSKe75d/mlUHrQ==} run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -8525,7 +8534,19 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.60.4': optional: true - '@rrweb/types@2.0.0-alpha.20': {} + '@rrweb/packer@2.0.1': + dependencies: + '@rrweb/types': 2.0.1 + fflate: 0.4.8 + + '@rrweb/replay@2.0.1': + dependencies: + '@rrweb/types': 2.0.1 + rrweb: 2.0.1 + + '@rrweb/types@2.0.1': {} + + '@rrweb/utils@2.0.1': {} '@schummar/icu-type-parser@1.21.5': {} @@ -11326,7 +11347,7 @@ snapshots: postcss@8.4.31: dependencies: - nanoid: 3.3.11 + nanoid: 3.3.12 picocolors: 1.1.1 source-map-js: 1.2.1 @@ -11726,27 +11747,30 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.4 fsevents: 2.3.3 - rrdom@0.1.7: + rrdom@2.0.1: dependencies: - rrweb-snapshot: 2.0.0-alpha.4 + rrweb-snapshot: 2.0.1 - rrweb-player@1.0.0-alpha.4: + rrweb-player@2.0.1: dependencies: + '@rrweb/packer': 2.0.1 + '@rrweb/replay': 2.0.1 '@tsconfig/svelte': 1.0.13 - rrweb: 2.0.0-alpha.4 - rrweb-snapshot@2.0.0-alpha.4: {} - - rrweb@2.0.0-alpha.4: + rrweb-snapshot@2.0.1: dependencies: - '@rrweb/types': 2.0.0-alpha.20 + postcss: 8.5.15 + + rrweb@2.0.1: + dependencies: + '@rrweb/types': 2.0.1 + '@rrweb/utils': 2.0.1 '@types/css-font-loading-module': 0.0.7 '@xstate/fsm': 1.6.5 base64-arraybuffer: 1.0.2 - fflate: 0.4.8 mitt: 3.0.1 - rrdom: 0.1.7 - rrweb-snapshot: 2.0.0-alpha.4 + rrdom: 2.0.1 + rrweb-snapshot: 2.0.1 run-parallel@1.2.0: dependencies: diff --git a/src/app/(main)/SideNav.tsx b/src/app/(main)/SideNav.tsx index c44011654..3f2b25390 100644 --- a/src/app/(main)/SideNav.tsx +++ b/src/app/(main)/SideNav.tsx @@ -74,7 +74,7 @@ export function SideNav(props: any) { backgroundColor="surface-base" border borderRadius - paddingX="2" + padding="2" flexGrow="1" minHeight="0" margin="2" @@ -137,7 +137,7 @@ export function SideNav(props: any) { )} - + diff --git a/src/app/api/record/route.ts b/src/app/api/record/route.ts index ad304a27c..7fa267b79 100644 --- a/src/app/api/record/route.ts +++ b/src/app/api/record/route.ts @@ -8,7 +8,7 @@ import { parseToken } from '@/lib/jwt'; import { fetchAccount, fetchTeam } from '@/lib/load'; import { getRecorderConfig } from '@/lib/recorder'; import { parseRequest } from '@/lib/request'; -import { badRequest, forbidden, json, serverError } from '@/lib/response'; +import { badRequest, forbidden, json, payloadTooLarge, serverError } from '@/lib/response'; import { getWebsite } from '@/queries/prisma'; import { saveRecording } from '@/queries/sql'; import { saveHeatmapEvents } from '@/queries/sql/heatmap/saveHeatmapEvents'; @@ -18,6 +18,8 @@ interface Cache { visitId: string; } +const MAX_RECORD_REQUEST_BYTES = 1_000_000; + const schema = z.discriminatedUnion('type', [ z.object({ type: z.literal('record'), @@ -73,8 +75,38 @@ function getUrlPath(url: string) { } } +async function getRequestBodySize(request: Request): Promise { + const contentLength = request.headers.get('content-length'); + + if (contentLength) { + const size = Number(contentLength); + + if (Number.isFinite(size)) { + return size; + } + } + + try { + const text = await request.clone().text(); + + return new TextEncoder().encode(text).length; + } catch { + return null; + } +} + export async function POST(request: Request) { try { + const requestBodySize = await getRequestBodySize(request); + + if (requestBodySize && requestBodySize > MAX_RECORD_REQUEST_BYTES) { + return payloadTooLarge({ + reason: 'payload_too_large', + maxBytes: MAX_RECORD_REQUEST_BYTES, + size: requestBodySize, + }); + } + const { body, error } = await parseRequest(request, schema, { skipAuth: true }); if (error) { diff --git a/src/lib/detect.test.ts b/src/lib/detect.test.ts index 6aaab223b..804c84029 100644 --- a/src/lib/detect.test.ts +++ b/src/lib/detect.test.ts @@ -1,8 +1,23 @@ -import { expect, test } from 'vitest'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { getLocation, hasBlockedIp } from './detect'; import { getIpAddress } from './ip'; const IP = '127.0.0.1'; +const isLocalhost = vi.mocked(await import('is-localhost-ip')); + +vi.mock('is-localhost-ip', () => ({ + default: vi.fn(), +})); + +beforeEach(() => { + vi.resetAllMocks(); + + delete process.env.CLIENT_IP_HEADER; + delete process.env.IGNORE_IP; + delete process.env.SKIP_LOCATION_HEADERS; +}); + test('getIpAddress: Custom header', () => { process.env.CLIENT_IP_HEADER = 'x-custom-ip-header'; @@ -20,3 +35,43 @@ test('getIpAddress: Standard header', () => { test('getIpAddress: No header', () => { expect(getIpAddress(new Headers())).toEqual(undefined); }); + +test('getLocation: returns null for malformed ip', async () => { + await expect( + getLocation( + 'not-an-ip', + new Headers({ + 'cf-ipcountry': 'US', + 'cf-region-code': 'CA', + 'cf-ipcity': 'Los Angeles', + }), + false, + ), + ).resolves.toEqual(null); +}); + +test('getLocation: treats localhost check errors as non-local', async () => { + isLocalhost.default.mockRejectedValue(new Error('DNS Lookup failed.')); + + await expect( + getLocation( + '8.8.8.8', + new Headers({ + 'cf-ipcountry': 'US', + 'cf-region-code': 'CA', + 'cf-ipcity': 'Los Angeles', + }), + false, + ), + ).resolves.toEqual({ + country: 'US', + region: 'US-CA', + city: 'Los Angeles', + }); +}); + +test('hasBlockedIp: returns false for malformed client ip with cidr block', () => { + process.env.IGNORE_IP = '10.0.0.0/8'; + + expect(hasBlockedIp('not-an-ip')).toBe(false); +}); diff --git a/src/lib/detect.ts b/src/lib/detect.ts index fb67571d9..fb36ba212 100644 --- a/src/lib/detect.ts +++ b/src/lib/detect.ts @@ -76,11 +76,19 @@ function decodeHeader(s: string | undefined | null): string | undefined | null { return Buffer.from(s, 'latin1').toString('utf-8'); } +async function isLocalIp(ip: string) { + try { + return await isLocalhost(ip); + } catch { + return false; + } +} + export async function getLocation(ip: string = '', headers: Headers, skipHeaders: boolean) { const cleanIp = stripPort(ip); // Ignore local or invalid ips - if (!cleanIp || !ipaddr.isValid(cleanIp) || (await isLocalhost(cleanIp))) { + if (!cleanIp || !ipaddr.isValid(cleanIp) || (await isLocalIp(cleanIp))) { return null; } diff --git a/src/lib/response.ts b/src/lib/response.ts index f1ad5c7bf..587802619 100644 --- a/src/lib/response.ts +++ b/src/lib/response.ts @@ -36,6 +36,15 @@ export function forbidden(error?: Record) { ); } +export function payloadTooLarge(error?: Record) { + return Response.json( + { + error: { message: 'Payload too large', code: 'payload-too-large', status: 413, ...error }, + }, + { status: 413 }, + ); +} + export function notFound(error?: Record) { return Response.json( { error: { message: 'Not found', code: 'not-found', status: 404, ...error } }, diff --git a/src/recorder/index.js b/src/recorder/index.js index b3262c79c..9f5950ec3 100644 --- a/src/recorder/index.js +++ b/src/recorder/index.js @@ -22,7 +22,8 @@ import { record } from 'rrweb'; const configEndpoint = `${hostBase}__RECORDER_CONFIG_ENDPOINT__`.replace('{websiteId}', website); const REPLAY_FLUSH_EVENT_COUNT = 100; - const REPLAY_FLUSH_INTERVAL = 10000; + const REPLAY_FLUSH_INTERVAL = 2000; + const REPLAY_MAX_PAYLOAD_SIZE = 900000; const HEATMAP_FLUSH_EVENT_COUNT = 20; const HEATMAP_FLUSH_INTERVAL = 5000; @@ -45,12 +46,8 @@ import { record } from 'rrweb'; const getSessionCache = () => window.umami?.getSession?.()?.cache; - const sendPayload = (type, payload, useKeepalive = false) => { - const cache = getSessionCache(); - - if (!cache) return; - - const body = JSON.stringify({ + const getPayloadBody = (type, payload) => + JSON.stringify({ type, payload: { website, @@ -58,7 +55,28 @@ import { record } from 'rrweb'; }, }); - const keepalive = useKeepalive && body.length < 60000; + const getPayloadSize = body => { + try { + return new Blob([body]).size; + } catch { + return body.length; + } + }; + + const getReplayPayloadSize = (events, timestamp) => + getPayloadSize(getPayloadBody('record', { events, timestamp })); + + const isReplayPayloadTooLarge = (events, timestamp) => + getReplayPayloadSize(events, timestamp) > REPLAY_MAX_PAYLOAD_SIZE; + + const sendPayload = (type, payload, useKeepalive = false) => { + const cache = getSessionCache(); + + if (!cache) return; + + const body = getPayloadBody(type, payload); + + const keepalive = useKeepalive && getPayloadSize(body) < 60000; return fetch(endpoint, { keepalive, @@ -72,20 +90,55 @@ import { record } from 'rrweb'; }).catch(() => {}); }; + const sendReplayEvents = (events, timestamp, useKeepalive = false) => { + let chunk = []; + let chunkOffset = 0; + + events.forEach(event => { + const candidate = [...chunk, event]; + + if (isReplayPayloadTooLarge(candidate, timestamp + chunkOffset)) { + if (chunk.length) { + sendPayload( + 'record', + { + events: chunk, + timestamp: timestamp + chunkOffset, + }, + useKeepalive, + ); + chunk = []; + chunkOffset += 1; + } + + if (isReplayPayloadTooLarge([event], timestamp + chunkOffset)) { + chunkOffset += 1; + return; + } + } + + chunk.push(event); + }); + + if (chunk.length) { + sendPayload( + 'record', + { + events: chunk, + timestamp: timestamp + chunkOffset, + }, + useKeepalive, + ); + } + }; + const flushReplay = (useKeepalive = false) => { if (!replayBuffer.length) return; const events = replayBuffer; replayBuffer = []; - sendPayload( - 'record', - { - events, - timestamp: Math.floor(Date.now() / 1000), - }, - useKeepalive, - ); + sendReplayEvents(events, Math.floor(Date.now() / 1000), useKeepalive); }; const flushHeatmap = (useKeepalive = false) => { @@ -104,11 +157,6 @@ import { record } from 'rrweb'; ); }; - const scheduleReplayFlush = () => { - if (replayFlushTimer) clearTimeout(replayFlushTimer); - replayFlushTimer = setTimeout(() => flushReplay(), REPLAY_FLUSH_INTERVAL); - }; - const scheduleHeatmapFlush = () => { if (heatmapFlushTimer) clearTimeout(heatmapFlushTimer); heatmapFlushTimer = setTimeout(() => flushHeatmap(), HEATMAP_FLUSH_INTERVAL); @@ -133,7 +181,7 @@ import { record } from 'rrweb'; replayStopped = true; - if (replayFlushTimer) clearTimeout(replayFlushTimer); + if (replayFlushTimer) clearInterval(replayFlushTimer); flushReplay(); if (replayStopFn) { @@ -267,6 +315,8 @@ import { record } from 'rrweb'; const beginReplayCapture = () => { replayStartTime = Date.now(); + replayFlushTimer = setInterval(() => flushReplay(), REPLAY_FLUSH_INTERVAL); + replayStopFn = record({ emit(event) { if (replayStopped) return; @@ -276,13 +326,21 @@ import { record } from 'rrweb'; return; } - replayBuffer.push(event); - - if (replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT) { + if ( + replayBuffer.length && + isReplayPayloadTooLarge([...replayBuffer, event], Math.floor(Date.now() / 1000)) + ) { flushReplay(); } - scheduleReplayFlush(); + replayBuffer.push(event); + + if ( + replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT || + isReplayPayloadTooLarge(replayBuffer, Math.floor(Date.now() / 1000)) + ) { + flushReplay(); + } }, ...getMaskConfig(maskLevel), inlineStylesheet: true, @@ -465,6 +523,10 @@ import { record } from 'rrweb'; } }); + window.addEventListener('pagehide', () => { + flushReplay(true); + }); + window.addEventListener('beforeunload', () => { flushReplay(true); });