lly/umami
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: harden analytics writes and hide internal server errors from client

Browse Source
This commit is contained in:
Francis Cao
2026-06-22 22:07:07 -07:00
parent 770b4d59d2
commit 129681ebd7
15 changed files with 217 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/api/auth/sso/route.ts
+1 -3
View File
@@ -11,9 +11,7 @@ export async function POST(request: Request) {
}
if (!redis.enabled) {
return serverError({
message: 'Redis is disabled',
});
return serverError('Redis is disabled');
}
const token = await saveAuth({ userId: auth.user.id }, 86400);
src/app/api/record/route.ts
+1 -6
View File
@@ -244,11 +244,6 @@ export async function POST(request: Request) {
return json({ ok: true });
} catch (e) {
const error = serializeError(e);
// eslint-disable-next-line no-console
console.log(error);
return serverError({ errorObject: error });
return serverError(e);
}
}
src/app/api/send/route.ts
+15 -24
View File
@@ -1,6 +1,5 @@
import { startOfHour } from 'date-fns';
import { isbot } from 'isbot';
import { serializeError } from 'serialize-error';
import { z } from 'zod';
import clickhouse from '@/lib/clickhouse';
import { CACHE_TOKEN_TYPE, COLLECTION_TYPE, EVENT_TYPE } from '@/lib/constants';
@@ -24,13 +23,10 @@ interface Cache {
// Reject strings whose first character is a spreadsheet formula trigger to
// prevent CSV formula injection in analytics exports (defense-in-depth).
const FORMULA_TRIGGER_RE = /^[=+\-@\t\r]/;
const safeStringParam = (maxLen: number) =>
z
.string()
.max(maxLen)
.refine(val => !FORMULA_TRIGGER_RE.test(val), {
message: 'Value must not start with =, +, -, @, tab, or carriage return',
});
const safeStringParam = () =>
z.string().refine(val => !FORMULA_TRIGGER_RE.test(val), {
message: 'Value must not start with =, +, -, @, tab, or carriage return',
});
const schema = z.object({
type: z.enum(['event', 'identify', 'performance']),
@@ -40,21 +36,21 @@ const schema = z.object({
link: z.uuid().optional(),
pixel: z.uuid().optional(),
data: anyObjectParam.optional(),
hostname: z.string().max(100).optional(),
language: z.string().max(35).optional(),
hostname: z.string().optional(),
language: z.string().optional(),
referrer: urlOrPathParam.optional(),
screen: z.string().max(11).optional(),
title: z.string().max(500).optional(),
screen: z.string().optional(),
title: z.string().optional(),
url: urlOrPathParam.optional(),
name: safeStringParam(50).optional(),
tag: safeStringParam(50).optional(),
name: safeStringParam().optional(),
tag: safeStringParam().optional(),
ip: z.string().optional(),
userAgent: z.string().optional(),
timestamp: z.coerce.number().int().optional(),
id: z.string().max(50).optional(),
browser: z.string().max(20).optional(),
os: z.string().max(20).optional(),
device: z.string().max(20).optional(),
id: z.string().optional(),
browser: z.string().optional(),
os: z.string().optional(),
device: z.string().optional(),
lcp: z.number().nonnegative().max(60000).optional(),
inp: z.number().nonnegative().max(60000).optional(),
cls: z.number().nonnegative().max(100).optional(),
@@ -326,11 +322,6 @@ export async function POST(request: Request) {
return json({ cache: token, sessionId, visitId });
} catch (e) {
const error = serializeError(e);
// eslint-disable-next-line no-console
console.log(error);
return serverError({ errorObject: error });
return serverError(e);
}
}
src/lib/constants.ts
+20 -6
View File
@@ -268,12 +268,26 @@ export const SHARE_ID_REGEX = /^[a-zA-Z0-9]{8,50}$/;
export const DATETIME_REGEX =
/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3})?(Z|\+[0-9]{2}:[0-9]{2})?$/;
export const URL_LENGTH = 500;
export const PAGE_TITLE_LENGTH = 500;
export const EVENT_NAME_LENGTH = 50;
export const TAG_LENGTH = 50;
export const HOSTNAME_LENGTH = 100;
export const FIELD_VALUE_LENGTH = 255;
export const FIELD_LENGTH = {
browser: 20,
os: 20,
device: 20,
screen: 11,
language: 35,
country: 2,
region: 20,
city: 50,
distinctId: 50,
url: 500,
pageTitle: 500,
eventName: 50,
tag: 50,
hostname: 100,
fieldValue: 255,
dataKey: 500,
stringValue: 500,
currency: 10,
} as const;
export const UTM_PARAMS = ['utm_campaign', 'utm_content', 'utm_medium', 'utm_source', 'utm_term'];
src/lib/data.test.ts
+17
View File
@@ -0,0 +1,17 @@
import { describe, expect, test } from 'vitest';
import { DATA_TYPE, FIELD_LENGTH } from './constants';
import { getStoredStringValue } from './data';
describe('getStoredStringValue', () => {
test('truncates oversized string values to the storage limit', () => {
expect(getStoredStringValue('x'.repeat(FIELD_LENGTH.stringValue + 25), DATA_TYPE.string)).toHaveLength(
FIELD_LENGTH.stringValue,
);
});
test('drops oversized array payloads instead of storing invalid truncated JSON', () => {
const oversizedArray = JSON.stringify([`x${'y'.repeat(FIELD_LENGTH.stringValue)}`]);
expect(getStoredStringValue(oversizedArray, DATA_TYPE.array)).toBeNull();
});
});
src/lib/data.ts
+12 -1
View File
@@ -1,4 +1,5 @@
import { DATA_TYPE, DATETIME_REGEX } from './constants';
import { DATA_TYPE, DATETIME_REGEX, FIELD_LENGTH } from './constants';
import { truncateString } from './format';
import type { DynamicDataType } from './types';
export interface KeyValueData {
@@ -49,6 +50,16 @@ export function getStringValue(value: string, dataType: number) {
return value;
}
export function getStoredStringValue(value: string, dataType: number) {
const stringValue = getStringValue(value, dataType);
if (dataType === DATA_TYPE.array && stringValue.length > FIELD_LENGTH.stringValue) {
return null;
}
return truncateString(stringValue, FIELD_LENGTH.stringValue);
}
export function createKeyValue(key: string, value: any): KeyValueData {
const type = getDataType(value);
let dataType: DynamicDataType;
src/lib/format.test.ts
+5
View File
@@ -37,3 +37,8 @@ test('stringToColor', () => {
expect(format.stringToColor('hello')).toBe('#d218e9');
expect(format.stringToColor('goodbye')).toBe('#11e956');
});
test('truncateString', () => {
expect(format.truncateString('hello', 3)).toBe('hel');
expect(format.truncateString(undefined, 3)).toBeUndefined();
});
src/lib/format.ts
+7
View File
@@ -118,3 +118,10 @@ export function formatLongCurrency(value: number, currency: string, locale = 'en
return formatCurrency(n, currency, locale);
}
export function truncateString<T extends string | null | undefined>(
value: T,
maxLength: number,
): T extends string ? string : T {
return (value ? value.substring(0, maxLength) : value) as T extends string ? string : T;
}
src/lib/response.test.ts
+32
View File
@@ -0,0 +1,32 @@
import { describe, expect, test, vi } from 'vitest';
import { serverError } from './response';
describe('serverError', () => {
test('does not expose internal error details in the response body', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
const response = serverError(new Error('database exploded'));
expect(await response.json()).toEqual({
error: {
message: 'Server error',
code: 'server-error',
status: 500,
},
});
logSpy.mockRestore();
});
test('allows intentional server error messages', async () => {
const response = serverError('Redis is disabled');
expect(await response.json()).toEqual({
error: {
message: 'Redis is disabled',
code: 'server-error',
status: 500,
},
});
});
});
src/lib/response.ts
+9 -3
View File
@@ -1,3 +1,5 @@
import { serializeError } from 'serialize-error';
export function ok() {
return Response.json({ ok: true });
}
@@ -52,14 +54,18 @@ export function notFound(error?: Record<string, any>) {
);
}
export function serverError(error?: Record<string, any>) {
export function serverError(error?: unknown) {
if (error && typeof error !== 'string') {
// eslint-disable-next-line no-console
console.log(serializeError(error));
}
return Response.json(
{
error: {
message: 'Server error',
message: typeof error === 'string' ? error : 'Server error',
code: 'server-error',
status: 500,
...error,
},
},
{ status: 500 },
src/queries/sql/events/saveEvent.ts
+59 -66
View File
@@ -1,14 +1,8 @@
import clickhouse from '@/lib/clickhouse';
import {
EVENT_NAME_LENGTH,
FIELD_VALUE_LENGTH,
HOSTNAME_LENGTH,
PAGE_TITLE_LENGTH,
TAG_LENGTH,
URL_LENGTH,
} from '@/lib/constants';
import { FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import { saveEventData } from './saveEventData';
@@ -76,10 +70,6 @@ export async function saveEvent(args: SaveEventArgs) {
});
}
function truncate(value: string | null | undefined, maxLength: number) {
return value ? value.substring(0, maxLength) : value;
}
async function relationalQuery({
websiteId,
sessionId,
@@ -121,27 +111,27 @@ async function relationalQuery({
websiteId,
sessionId,
visitId,
urlPath: truncate(urlPath, URL_LENGTH),
urlQuery: truncate(urlQuery, URL_LENGTH),
utmSource: truncate(utmSource, FIELD_VALUE_LENGTH),
utmMedium: truncate(utmMedium, FIELD_VALUE_LENGTH),
utmCampaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
utmContent: truncate(utmContent, FIELD_VALUE_LENGTH),
utmTerm: truncate(utmTerm, FIELD_VALUE_LENGTH),
referrerPath: truncate(referrerPath, URL_LENGTH),
referrerQuery: truncate(referrerQuery, URL_LENGTH),
referrerDomain: truncate(referrerDomain, URL_LENGTH),
pageTitle: truncate(pageTitle, PAGE_TITLE_LENGTH),
gclid: truncate(gclid, FIELD_VALUE_LENGTH),
fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
lifatid: truncate(lifatid, FIELD_VALUE_LENGTH),
twclid: truncate(twclid, FIELD_VALUE_LENGTH),
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
urlQuery: truncateString(urlQuery, FIELD_LENGTH.url),
utmSource: truncateString(utmSource, FIELD_LENGTH.fieldValue),
utmMedium: truncateString(utmMedium, FIELD_LENGTH.fieldValue),
utmCampaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue),
utmContent: truncateString(utmContent, FIELD_LENGTH.fieldValue),
utmTerm: truncateString(utmTerm, FIELD_LENGTH.fieldValue),
referrerPath: truncateString(referrerPath, FIELD_LENGTH.url),
referrerQuery: truncateString(referrerQuery, FIELD_LENGTH.url),
referrerDomain: truncateString(referrerDomain, FIELD_LENGTH.url),
pageTitle: truncateString(pageTitle, FIELD_LENGTH.pageTitle),
gclid: truncateString(gclid, FIELD_LENGTH.fieldValue),
fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue),
msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue),
ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue),
lifatid: truncateString(lifatid, FIELD_LENGTH.fieldValue),
twclid: truncateString(twclid, FIELD_LENGTH.fieldValue),
eventType,
eventName: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
tag: truncate(tag, TAG_LENGTH),
hostname: truncate(hostname, HOSTNAME_LENGTH),
eventName: truncateString(eventName, FIELD_LENGTH.eventName) ?? null,
tag: truncateString(tag, FIELD_LENGTH.tag),
hostname: truncateString(hostname, FIELD_LENGTH.hostname),
lcp,
inp,
cls,
@@ -156,8 +146,8 @@ async function relationalQuery({
websiteId,
sessionId,
eventId: websiteEventId,
urlPath: truncate(urlPath, URL_LENGTH),
eventName: truncate(eventName, EVENT_NAME_LENGTH),
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
eventData,
createdAt,
});
@@ -169,7 +159,7 @@ async function relationalQuery({
websiteId,
sessionId,
eventId: websiteEventId,
eventName: truncate(eventName, EVENT_NAME_LENGTH),
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
currency,
revenue,
createdAt,
@@ -229,37 +219,40 @@ async function clickhouseQuery({
session_id: sessionId,
visit_id: visitId,
event_id: eventId,
country: country,
region: country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
city: city,
url_path: truncate(urlPath, URL_LENGTH),
url_query: truncate(urlQuery, URL_LENGTH),
utm_source: truncate(utmSource, FIELD_VALUE_LENGTH),
utm_medium: truncate(utmMedium, FIELD_VALUE_LENGTH),
utm_campaign: truncate(utmCampaign, FIELD_VALUE_LENGTH),
utm_content: truncate(utmContent, FIELD_VALUE_LENGTH),
utm_term: truncate(utmTerm, FIELD_VALUE_LENGTH),
referrer_path: truncate(referrerPath, URL_LENGTH),
referrer_query: truncate(referrerQuery, URL_LENGTH),
referrer_domain: truncate(referrerDomain, URL_LENGTH),
page_title: truncate(pageTitle, PAGE_TITLE_LENGTH),
gclid: truncate(gclid, FIELD_VALUE_LENGTH),
fbclid: truncate(fbclid, FIELD_VALUE_LENGTH),
msclkid: truncate(msclkid, FIELD_VALUE_LENGTH),
ttclid: truncate(ttclid, FIELD_VALUE_LENGTH),
li_fat_id: truncate(lifatid, FIELD_VALUE_LENGTH),
twclid: truncate(twclid, FIELD_VALUE_LENGTH),
region: truncateString(
country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
FIELD_LENGTH.region,
),
city: truncateString(city, FIELD_LENGTH.city),
url_path: truncateString(urlPath, FIELD_LENGTH.url),
url_query: truncateString(urlQuery, FIELD_LENGTH.url),
utm_source: truncateString(utmSource, FIELD_LENGTH.fieldValue),
utm_medium: truncateString(utmMedium, FIELD_LENGTH.fieldValue),
utm_campaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue),
utm_content: truncateString(utmContent, FIELD_LENGTH.fieldValue),
utm_term: truncateString(utmTerm, FIELD_LENGTH.fieldValue),
referrer_path: truncateString(referrerPath, FIELD_LENGTH.url),
referrer_query: truncateString(referrerQuery, FIELD_LENGTH.url),
referrer_domain: truncateString(referrerDomain, FIELD_LENGTH.url),
page_title: truncateString(pageTitle, FIELD_LENGTH.pageTitle),
gclid: truncateString(gclid, FIELD_LENGTH.fieldValue),
fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue),
msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue),
ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue),
li_fat_id: truncateString(lifatid, FIELD_LENGTH.fieldValue),
twclid: truncateString(twclid, FIELD_LENGTH.fieldValue),
event_type: eventType,
event_name: truncate(eventName, EVENT_NAME_LENGTH) ?? null,
tag: truncate(tag, TAG_LENGTH),
distinct_id: distinctId,
event_name: truncateString(eventName, FIELD_LENGTH.eventName) ?? null,
tag: truncateString(tag, FIELD_LENGTH.tag),
distinct_id: truncateString(distinctId, FIELD_LENGTH.distinctId),
created_at: getUTCString(createdAt),
browser: browser,
os: os,
device: device,
screen: screen,
language: language,
hostname: truncate(hostname, HOSTNAME_LENGTH),
browser: truncateString(browser, FIELD_LENGTH.browser),
os: truncateString(os, FIELD_LENGTH.os),
device: truncateString(device, FIELD_LENGTH.device),
screen: truncateString(screen, FIELD_LENGTH.screen),
language: truncateString(language, FIELD_LENGTH.language),
hostname: truncateString(hostname, FIELD_LENGTH.hostname),
country: truncateString(country, FIELD_LENGTH.country),
lcp: lcp,
inp: inp,
cls: cls,
@@ -278,8 +271,8 @@ async function clickhouseQuery({
websiteId,
sessionId,
eventId,
urlPath: truncate(urlPath, URL_LENGTH),
eventName: truncate(eventName, EVENT_NAME_LENGTH),
urlPath: truncateString(urlPath, FIELD_LENGTH.url),
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
eventData,
createdAt,
});
src/queries/sql/events/saveEventData.ts
+9 -8
View File
@@ -1,8 +1,9 @@
import clickhouse from '@/lib/clickhouse';
import { DATA_TYPE } from '@/lib/constants';
import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import { flattenJSON, getStoredStringValue } from '@/lib/data';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import type { DynamicData } from '@/lib/types';
@@ -34,8 +35,8 @@ async function relationalQuery(data: SaveEventDataArgs) {
id: uuid(),
websiteEventId: eventId,
websiteId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
dataKey: truncateString(a.key, FIELD_LENGTH.dataKey),
stringValue: getStoredStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
@@ -60,11 +61,11 @@ async function clickhouseQuery(data: SaveEventDataArgs) {
website_id: websiteId,
session_id: sessionId,
event_id: eventId,
url_path: urlPath,
event_name: eventName,
data_key: key,
url_path: truncateString(urlPath, FIELD_LENGTH.url),
event_name: truncateString(eventName, FIELD_LENGTH.eventName),
data_key: truncateString(key, FIELD_LENGTH.dataKey),
data_type: dataType,
string_value: getStringValue(value, dataType),
string_value: getStoredStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
created_at: getUTCString(createdAt),
src/queries/sql/events/saveRevenue.ts
+4 -2
View File
@@ -1,5 +1,7 @@
import { FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import prisma from '@/lib/prisma';
export interface SaveRevenueArgs {
@@ -27,8 +29,8 @@ async function relationalQuery(data: SaveRevenueArgs) {
websiteId,
sessionId,
eventId,
eventName,
currency,
eventName: truncateString(eventName, FIELD_LENGTH.eventName),
currency: truncateString(currency, FIELD_LENGTH.currency),
revenue,
createdAt,
},
src/queries/sql/sessions/createSession.ts
+15 -1
View File
@@ -1,10 +1,24 @@
import type { Prisma } from '@/generated/prisma/client';
import { FIELD_LENGTH } from '@/lib/constants';
import { truncateString } from '@/lib/format';
import prisma from '@/lib/prisma';
const FUNCTION_NAME = 'createSession';
export async function createSession(data: Prisma.SessionCreateInput) {
const { rawQuery } = prisma;
const normalizedData: Prisma.SessionCreateInput = {
...data,
browser: truncateString(data.browser, FIELD_LENGTH.browser),
os: truncateString(data.os, FIELD_LENGTH.os),
device: truncateString(data.device, FIELD_LENGTH.device),
screen: truncateString(data.screen, FIELD_LENGTH.screen),
language: truncateString(data.language, FIELD_LENGTH.language),
country: truncateString(data.country, FIELD_LENGTH.country),
region: truncateString(data.region, FIELD_LENGTH.region),
city: truncateString(data.city, FIELD_LENGTH.city),
distinctId: truncateString(data.distinctId, FIELD_LENGTH.distinctId),
};
await rawQuery(
`
@@ -38,7 +52,7 @@ export async function createSession(data: Prisma.SessionCreateInput) {
)
on conflict (session_id) do nothing
`,
data,
normalizedData,
FUNCTION_NAME,
);
}
src/queries/sql/sessions/saveSessionData.ts
+11 -8
View File
@@ -1,8 +1,9 @@
import clickhouse from '@/lib/clickhouse';
import { DATA_TYPE } from '@/lib/constants';
import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import { flattenJSON, getStoredStringValue } from '@/lib/data';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { truncateString } from '@/lib/format';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import type { DynamicData } from '@/lib/types';
@@ -32,17 +33,18 @@ export async function relationalQuery({
const { client } = prisma;
const jsonKeys = flattenJSON(sessionData);
const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId);
const flattenedData = jsonKeys.map(a => ({
id: uuid(),
websiteId,
sessionId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
dataKey: truncateString(a.key, FIELD_LENGTH.dataKey),
stringValue: getStoredStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
distinctId,
distinctId: normalizedDistinctId,
createdAt,
}));
@@ -79,17 +81,18 @@ async function clickhouseQuery({
const { sendMessage } = kafka;
const jsonKeys = flattenJSON(sessionData);
const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId);
const messages = jsonKeys.map(({ key, value, dataType }) => {
return {
website_id: websiteId,
session_id: sessionId,
data_key: key,
data_key: truncateString(key, FIELD_LENGTH.dataKey),
data_type: dataType,
string_value: getStringValue(value, dataType),
string_value: getStoredStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
distinct_id: distinctId,
distinct_id: normalizedDistinctId,
created_at: getUTCString(createdAt),
};
});