Upgrade rrweb and chunk replay payloads

package.json

@@ -112,8 +112,8 @@
     "react-window": "^2.2.7",
     "redis": "^5.12.1",
     "request-ip": "^3.3.0",
-    "rrweb": "2.0.0-alpha.4",
+    "rrweb": "^2.0.1",
-    "rrweb-player": "1.0.0-alpha.4",
+    "rrweb-player": "^2.0.1",
     "semver": "^7.8.0",
     "serialize-error": "^13.0.1",
     "thenby": "^1.4.1",

pnpm-lock.yaml

@@ -186,11 +186,11 @@ importers:
         specifier: ^3.3.0
         version: 3.3.0
       rrweb:
-        specifier: 2.0.0-alpha.4
+        specifier: ^2.0.1
-        version: 2.0.0-alpha.4
+        version: 2.0.1
       rrweb-player:
-        specifier: 1.0.0-alpha.4
+        specifier: ^2.0.1
-        version: 1.0.0-alpha.4
+        version: 2.0.1
       semver:
         specifier: ^7.8.0
         version: 7.8.0
@@ -2633,8 +2633,17 @@ packages:
     cpu: [x64]
     os: [win32]
 
-  '@rrweb/types@2.0.0-alpha.20':
+  '@rrweb/packer@2.0.1':
-    resolution: {integrity: sha512-RbnDgKxA/odwB1R4gF7eUUj+rdSrq6ROQJsnMw7MIsGzlbSYvJeZN8YY4XqU0G6sKJvXI6bSzk7w/G94jNwzhw==}
+    resolution: {integrity: sha512-2U8RkVe+ZK3gTtmKjJKjoh6QL1Q6wIlt3sBTJeCPERLr9e0JIimNrZ1+bog1L30CTVP3ULuDf5Ba7RZO+QNcHg==}
+
+  '@rrweb/replay@2.0.1':
+    resolution: {integrity: sha512-UrZONirb8o4t0urg1aJ8aMSWhNHvJ3a2usuVbPJIm/edyFbwTvvY9sg/rrXccyXOntFhhp73R5nCretODRZcCw==}
+
+  '@rrweb/types@2.0.1':
+    resolution: {integrity: sha512-x8cNaKFxdvS0/5IfKOj1xG8Htnw1fUwqoPqgP61bOaWqZL6t9UtRPpO9ZlMB2z+FiTxg9zlFFL+Da4ZpFuhoiA==}
+
+  '@rrweb/utils@2.0.1':
+    resolution: {integrity: sha512-t+b+4rB23/e4EpoGMdPUr6cKtDsFuOb3LxA9IHE8e3A/Xo0nMUfH0bkwqdpUNa/VGDOdiGeG3bhHjuO9zWX54g==}
 
   '@schummar/icu-type-parser@1.21.5':
     resolution: {integrity: sha512-bXHSaW5jRTmke9Vd0h5P7BtWZG9Znqb8gSDxZnxaGSJnGwPLDPfS+3g0BKzeWqzgZPsIVZkM7m2tbo18cm5HBw==}
@@ -5658,17 +5667,17 @@ packages:
     engines: {node: '>=18.0.0', npm: '>=8.0.0'}
     hasBin: true
 
-  rrdom@0.1.7:
+  rrdom@2.0.1:
-    resolution: {integrity: sha512-ZLd8f14z9pUy2Hk9y636cNv5Y2BMnNEY99wxzW9tD2BLDfe1xFxtLjB4q/xCBYo6HRe0wofzKzjm4JojmpBfFw==}
+    resolution: {integrity: sha512-z8GdQWFmVfa8GpcGkAqfBXseb89z0ciTPSBkID0qr0i6KHCye1MdQ4VoXFf0Ejp3ELe/c7AObu4zsbSbMT+hiw==}
 
-  rrweb-player@1.0.0-alpha.4:
+  rrweb-player@2.0.1:
-    resolution: {integrity: sha512-Wlmn9GZ5Fdqa37vd3TzsYdLl/JWEvXNUrLCrYpnOwEgmY409HwVIvvA5aIo7k582LoKgdRCsB87N+f0oWAR0Kg==}
+    resolution: {integrity: sha512-v8PVnOy7Q3kXqWriGuzZo4ndGEV06nCMRkH/jSHweJfTLwpLqb2evgMj0S5FwQLsN2X1VM+L2/i7QWGJfgj9zw==}
 
-  rrweb-snapshot@2.0.0-alpha.4:
+  rrweb-snapshot@2.0.1:
-    resolution: {integrity: sha512-KQ2OtPpXO5jLYqg1OnXS/Hf+EzqnZyP5A+XPqBCjYpj3XIje/Od4gdUwjbFo3cVuWq5Cw5Y1d3/xwgIS7/XpQQ==}
+    resolution: {integrity: sha512-H8TA29ct2tpYtr5oaB0N+ksRY9bhEErfAYcKbce4+w8lqEUQChpKZd41HpBJLkp04b5QUfWUg4v6r9Okf8r7vA==}
 
-  rrweb@2.0.0-alpha.4:
+  rrweb@2.0.1:
-    resolution: {integrity: sha512-wEHUILbxDPcNwkM3m4qgPgXAiBJyqCbbOHyVoNEVBJzHszWEFYyTbrZqUdeb1EfmTRC2PsumCIkVcomJ/xcOzA==}
+    resolution: {integrity: sha512-MQYzOtI7aZft9ffDMT6rmRSAj6hO4W6T7iA4eB9Mtcuiv3M7DbnnxDMrcPW0sQXepgYN46YhfSKe75d/mlUHrQ==}
 
   run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
@@ -8525,7 +8534,19 @@ snapshots:
   '@rollup/rollup-win32-x64-msvc@4.60.4':
     optional: true
 
-  '@rrweb/types@2.0.0-alpha.20': {}
+  '@rrweb/packer@2.0.1':
+    dependencies:
+      '@rrweb/types': 2.0.1
+      fflate: 0.4.8
+
+  '@rrweb/replay@2.0.1':
+    dependencies:
+      '@rrweb/types': 2.0.1
+      rrweb: 2.0.1
+
+  '@rrweb/types@2.0.1': {}
+
+  '@rrweb/utils@2.0.1': {}
 
   '@schummar/icu-type-parser@1.21.5': {}
 
@@ -11326,7 +11347,7 @@ snapshots:
 
   postcss@8.4.31:
     dependencies:
-      nanoid: 3.3.11
+      nanoid: 3.3.12
       picocolors: 1.1.1
       source-map-js: 1.2.1
 
@@ -11726,27 +11747,30 @@ snapshots:
       '@rollup/rollup-win32-x64-msvc': 4.60.4
       fsevents: 2.3.3
 
-  rrdom@0.1.7:
+  rrdom@2.0.1:
     dependencies:
-      rrweb-snapshot: 2.0.0-alpha.4
+      rrweb-snapshot: 2.0.1
 
-  rrweb-player@1.0.0-alpha.4:
+  rrweb-player@2.0.1:
     dependencies:
+      '@rrweb/packer': 2.0.1
+      '@rrweb/replay': 2.0.1
       '@tsconfig/svelte': 1.0.13
-      rrweb: 2.0.0-alpha.4
 
-  rrweb-snapshot@2.0.0-alpha.4: {}
+  rrweb-snapshot@2.0.1:
-
-  rrweb@2.0.0-alpha.4:
     dependencies:
-      '@rrweb/types': 2.0.0-alpha.20
+      postcss: 8.5.15
+
+  rrweb@2.0.1:
+    dependencies:
+      '@rrweb/types': 2.0.1
+      '@rrweb/utils': 2.0.1
       '@types/css-font-loading-module': 0.0.7
       '@xstate/fsm': 1.6.5
       base64-arraybuffer: 1.0.2
-      fflate: 0.4.8
       mitt: 3.0.1
-      rrdom: 0.1.7
+      rrdom: 2.0.1
-      rrweb-snapshot: 2.0.0-alpha.4
+      rrweb-snapshot: 2.0.1
 
   run-parallel@1.2.0:
     dependencies:

src/app/(main)/SideNav.tsx

@@ -74,7 +74,7 @@ export function SideNav(props: any) {
       backgroundColor="surface-base"
       border
       borderRadius
-      paddingX="2"
+      padding="2"
       flexGrow="1"
       minHeight="0"
       margin="2"
@@ -137,7 +137,7 @@ export function SideNav(props: any) {
           </Column>
         )}
       </Column>
-      <Row marginBottom="4" paddingTop="2">
+      <Row paddingTop="2">
         <UserButton showText={!isCollapsed} />
       </Row>
     </Column>

src/app/api/record/route.ts

@@ -8,7 +8,7 @@ import { parseToken } from '@/lib/jwt';
 import { fetchAccount, fetchTeam } from '@/lib/load';
 import { getRecorderConfig } from '@/lib/recorder';
 import { parseRequest } from '@/lib/request';
-import { badRequest, forbidden, json, serverError } from '@/lib/response';
+import { badRequest, forbidden, json, payloadTooLarge, serverError } from '@/lib/response';
 import { getWebsite } from '@/queries/prisma';
 import { saveRecording } from '@/queries/sql';
 import { saveHeatmapEvents } from '@/queries/sql/heatmap/saveHeatmapEvents';
@@ -18,6 +18,8 @@ interface Cache {
   visitId: string;
 }
 
+const MAX_RECORD_REQUEST_BYTES = 1_000_000;
+
 const schema = z.discriminatedUnion('type', [
   z.object({
     type: z.literal('record'),
@@ -73,8 +75,38 @@ function getUrlPath(url: string) {
   }
 }
 
+async function getRequestBodySize(request: Request): Promise<number | null> {
+  const contentLength = request.headers.get('content-length');
+
+  if (contentLength) {
+    const size = Number(contentLength);
+
+    if (Number.isFinite(size)) {
+      return size;
+    }
+  }
+
+  try {
+    const text = await request.clone().text();
+
+    return new TextEncoder().encode(text).length;
+  } catch {
+    return null;
+  }
+}
+
 export async function POST(request: Request) {
   try {
+    const requestBodySize = await getRequestBodySize(request);
+
+    if (requestBodySize && requestBodySize > MAX_RECORD_REQUEST_BYTES) {
+      return payloadTooLarge({
+        reason: 'payload_too_large',
+        maxBytes: MAX_RECORD_REQUEST_BYTES,
+        size: requestBodySize,
+      });
+    }
+
     const { body, error } = await parseRequest(request, schema, { skipAuth: true });
 
     if (error) {

src/lib/detect.test.ts

@@ -1,8 +1,23 @@
-import { expect, test } from 'vitest';
+import { beforeEach, expect, test, vi } from 'vitest';
+import { getLocation, hasBlockedIp } from './detect';
 import { getIpAddress } from './ip';
 
 const IP = '127.0.0.1';
 
+const isLocalhost = vi.mocked(await import('is-localhost-ip'));
+
+vi.mock('is-localhost-ip', () => ({
+  default: vi.fn(),
+}));
+
+beforeEach(() => {
+  vi.resetAllMocks();
+
+  delete process.env.CLIENT_IP_HEADER;
+  delete process.env.IGNORE_IP;
+  delete process.env.SKIP_LOCATION_HEADERS;
+});
+
 test('getIpAddress: Custom header', () => {
   process.env.CLIENT_IP_HEADER = 'x-custom-ip-header';
 
@@ -20,3 +35,43 @@ test('getIpAddress: Standard header', () => {
 test('getIpAddress: No header', () => {
   expect(getIpAddress(new Headers())).toEqual(undefined);
 });
+
+test('getLocation: returns null for malformed ip', async () => {
+  await expect(
+    getLocation(
+      'not-an-ip',
+      new Headers({
+        'cf-ipcountry': 'US',
+        'cf-region-code': 'CA',
+        'cf-ipcity': 'Los Angeles',
+      }),
+      false,
+    ),
+  ).resolves.toEqual(null);
+});
+
+test('getLocation: treats localhost check errors as non-local', async () => {
+  isLocalhost.default.mockRejectedValue(new Error('DNS Lookup failed.'));
+
+  await expect(
+    getLocation(
+      '8.8.8.8',
+      new Headers({
+        'cf-ipcountry': 'US',
+        'cf-region-code': 'CA',
+        'cf-ipcity': 'Los Angeles',
+      }),
+      false,
+    ),
+  ).resolves.toEqual({
+    country: 'US',
+    region: 'US-CA',
+    city: 'Los Angeles',
+  });
+});
+
+test('hasBlockedIp: returns false for malformed client ip with cidr block', () => {
+  process.env.IGNORE_IP = '10.0.0.0/8';
+
+  expect(hasBlockedIp('not-an-ip')).toBe(false);
+});

src/lib/detect.ts

@@ -76,11 +76,19 @@ function decodeHeader(s: string | undefined | null): string | undefined | null {
   return Buffer.from(s, 'latin1').toString('utf-8');
 }
 
+async function isLocalIp(ip: string) {
+  try {
+    return await isLocalhost(ip);
+  } catch {
+    return false;
+  }
+}
+
 export async function getLocation(ip: string = '', headers: Headers, skipHeaders: boolean) {
   const cleanIp = stripPort(ip);
 
   // Ignore local or invalid ips
-  if (!cleanIp || !ipaddr.isValid(cleanIp) || (await isLocalhost(cleanIp))) {
+  if (!cleanIp || !ipaddr.isValid(cleanIp) || (await isLocalIp(cleanIp))) {
     return null;
   }
 

src/lib/response.ts

@@ -36,6 +36,15 @@ export function forbidden(error?: Record<string, any>) {
   );
 }
 
+export function payloadTooLarge(error?: Record<string, any>) {
+  return Response.json(
+    {
+      error: { message: 'Payload too large', code: 'payload-too-large', status: 413, ...error },
+    },
+    { status: 413 },
+  );
+}
+
 export function notFound(error?: Record<string, any>) {
   return Response.json(
     { error: { message: 'Not found', code: 'not-found', status: 404, ...error } },

src/recorder/index.js

@@ -22,7 +22,8 @@ import { record } from 'rrweb';
   const configEndpoint = `${hostBase}__RECORDER_CONFIG_ENDPOINT__`.replace('{websiteId}', website);
 
   const REPLAY_FLUSH_EVENT_COUNT = 100;
-  const REPLAY_FLUSH_INTERVAL = 10000;
+  const REPLAY_FLUSH_INTERVAL = 2000;
+  const REPLAY_MAX_PAYLOAD_SIZE = 900000;
   const HEATMAP_FLUSH_EVENT_COUNT = 20;
   const HEATMAP_FLUSH_INTERVAL = 5000;
 
@@ -45,12 +46,8 @@ import { record } from 'rrweb';
 
   const getSessionCache = () => window.umami?.getSession?.()?.cache;
 
-  const sendPayload = (type, payload, useKeepalive = false) => {
+  const getPayloadBody = (type, payload) =>
-    const cache = getSessionCache();
+    JSON.stringify({
-
-    if (!cache) return;
-
-    const body = JSON.stringify({
       type,
       payload: {
         website,
@@ -58,7 +55,28 @@ import { record } from 'rrweb';
       },
     });
 
-    const keepalive = useKeepalive && body.length < 60000;
+  const getPayloadSize = body => {
+    try {
+      return new Blob([body]).size;
+    } catch {
+      return body.length;
+    }
+  };
+
+  const getReplayPayloadSize = (events, timestamp) =>
+    getPayloadSize(getPayloadBody('record', { events, timestamp }));
+
+  const isReplayPayloadTooLarge = (events, timestamp) =>
+    getReplayPayloadSize(events, timestamp) > REPLAY_MAX_PAYLOAD_SIZE;
+
+  const sendPayload = (type, payload, useKeepalive = false) => {
+    const cache = getSessionCache();
+
+    if (!cache) return;
+
+    const body = getPayloadBody(type, payload);
+
+    const keepalive = useKeepalive && getPayloadSize(body) < 60000;
 
     return fetch(endpoint, {
       keepalive,
@@ -72,20 +90,55 @@ import { record } from 'rrweb';
     }).catch(() => {});
   };
 
+  const sendReplayEvents = (events, timestamp, useKeepalive = false) => {
+    let chunk = [];
+    let chunkOffset = 0;
+
+    events.forEach(event => {
+      const candidate = [...chunk, event];
+
+      if (isReplayPayloadTooLarge(candidate, timestamp + chunkOffset)) {
+        if (chunk.length) {
+          sendPayload(
+            'record',
+            {
+              events: chunk,
+              timestamp: timestamp + chunkOffset,
+            },
+            useKeepalive,
+          );
+          chunk = [];
+          chunkOffset += 1;
+        }
+
+        if (isReplayPayloadTooLarge([event], timestamp + chunkOffset)) {
+          chunkOffset += 1;
+          return;
+        }
+      }
+
+      chunk.push(event);
+    });
+
+    if (chunk.length) {
+      sendPayload(
+        'record',
+        {
+          events: chunk,
+          timestamp: timestamp + chunkOffset,
+        },
+        useKeepalive,
+      );
+    }
+  };
+
   const flushReplay = (useKeepalive = false) => {
     if (!replayBuffer.length) return;
 
     const events = replayBuffer;
     replayBuffer = [];
 
-    sendPayload(
+    sendReplayEvents(events, Math.floor(Date.now() / 1000), useKeepalive);
-      'record',
-      {
-        events,
-        timestamp: Math.floor(Date.now() / 1000),
-      },
-      useKeepalive,
-    );
   };
 
   const flushHeatmap = (useKeepalive = false) => {
@@ -104,11 +157,6 @@ import { record } from 'rrweb';
     );
   };
 
-  const scheduleReplayFlush = () => {
-    if (replayFlushTimer) clearTimeout(replayFlushTimer);
-    replayFlushTimer = setTimeout(() => flushReplay(), REPLAY_FLUSH_INTERVAL);
-  };
-
   const scheduleHeatmapFlush = () => {
     if (heatmapFlushTimer) clearTimeout(heatmapFlushTimer);
     heatmapFlushTimer = setTimeout(() => flushHeatmap(), HEATMAP_FLUSH_INTERVAL);
@@ -133,7 +181,7 @@ import { record } from 'rrweb';
 
     replayStopped = true;
 
-    if (replayFlushTimer) clearTimeout(replayFlushTimer);
+    if (replayFlushTimer) clearInterval(replayFlushTimer);
     flushReplay();
 
     if (replayStopFn) {
@@ -267,6 +315,8 @@ import { record } from 'rrweb';
   const beginReplayCapture = () => {
     replayStartTime = Date.now();
 
+    replayFlushTimer = setInterval(() => flushReplay(), REPLAY_FLUSH_INTERVAL);
+
     replayStopFn = record({
       emit(event) {
         if (replayStopped) return;
@@ -276,13 +326,21 @@ import { record } from 'rrweb';
           return;
         }
 
-        replayBuffer.push(event);
+        if (
-
+          replayBuffer.length &&
-        if (replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT) {
+          isReplayPayloadTooLarge([...replayBuffer, event], Math.floor(Date.now() / 1000))
+        ) {
           flushReplay();
         }
 
-        scheduleReplayFlush();
+        replayBuffer.push(event);
+
+        if (
+          replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT ||
+          isReplayPayloadTooLarge(replayBuffer, Math.floor(Date.now() / 1000))
+        ) {
+          flushReplay();
+        }
       },
       ...getMaskConfig(maskLevel),
       inlineStylesheet: true,
@@ -465,6 +523,10 @@ import { record } from 'rrweb';
       }
     });
 
+    window.addEventListener('pagehide', () => {
+      flushReplay(true);
+    });
+
     window.addEventListener('beforeunload', () => {
       flushReplay(true);
     });