diff --git a/src/app/api/auth/sso/route.ts b/src/app/api/auth/sso/route.ts index f82228695..bf4bc2209 100644 --- a/src/app/api/auth/sso/route.ts +++ b/src/app/api/auth/sso/route.ts @@ -11,9 +11,7 @@ export async function POST(request: Request) { } if (!redis.enabled) { - return serverError({ - message: 'Redis is disabled', - }); + return serverError('Redis is disabled'); } const token = await saveAuth({ userId: auth.user.id }, 86400); diff --git a/src/app/api/record/route.ts b/src/app/api/record/route.ts index ade6aa0c2..479bba1c8 100644 --- a/src/app/api/record/route.ts +++ b/src/app/api/record/route.ts @@ -244,11 +244,6 @@ export async function POST(request: Request) { return json({ ok: true }); } catch (e) { - const error = serializeError(e); - - // eslint-disable-next-line no-console - console.log(error); - - return serverError({ errorObject: error }); + return serverError(e); } } diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index 532836d97..6d2086176 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -1,6 +1,5 @@ import { startOfHour } from 'date-fns'; import { isbot } from 'isbot'; -import { serializeError } from 'serialize-error'; import { z } from 'zod'; import clickhouse from '@/lib/clickhouse'; import { CACHE_TOKEN_TYPE, COLLECTION_TYPE, EVENT_TYPE } from '@/lib/constants'; @@ -24,13 +23,10 @@ interface Cache { // Reject strings whose first character is a spreadsheet formula trigger to // prevent CSV formula injection in analytics exports (defense-in-depth). const FORMULA_TRIGGER_RE = /^[=+\-@\t\r]/; -const safeStringParam = (maxLen: number) => - z - .string() - .max(maxLen) - .refine(val => !FORMULA_TRIGGER_RE.test(val), { - message: 'Value must not start with =, +, -, @, tab, or carriage return', - }); +const safeStringParam = () => + z.string().refine(val => !FORMULA_TRIGGER_RE.test(val), { + message: 'Value must not start with =, +, -, @, tab, or carriage return', + }); const schema = z.object({ type: z.enum(['event', 'identify', 'performance']), @@ -40,21 +36,21 @@ const schema = z.object({ link: z.uuid().optional(), pixel: z.uuid().optional(), data: anyObjectParam.optional(), - hostname: z.string().max(100).optional(), - language: z.string().max(35).optional(), + hostname: z.string().optional(), + language: z.string().optional(), referrer: urlOrPathParam.optional(), - screen: z.string().max(11).optional(), - title: z.string().max(500).optional(), + screen: z.string().optional(), + title: z.string().optional(), url: urlOrPathParam.optional(), - name: safeStringParam(50).optional(), - tag: safeStringParam(50).optional(), + name: safeStringParam().optional(), + tag: safeStringParam().optional(), ip: z.string().optional(), userAgent: z.string().optional(), timestamp: z.coerce.number().int().optional(), - id: z.string().max(50).optional(), - browser: z.string().max(20).optional(), - os: z.string().max(20).optional(), - device: z.string().max(20).optional(), + id: z.string().optional(), + browser: z.string().optional(), + os: z.string().optional(), + device: z.string().optional(), lcp: z.number().nonnegative().max(60000).optional(), inp: z.number().nonnegative().max(60000).optional(), cls: z.number().nonnegative().max(100).optional(), @@ -326,11 +322,6 @@ export async function POST(request: Request) { return json({ cache: token, sessionId, visitId }); } catch (e) { - const error = serializeError(e); - - // eslint-disable-next-line no-console - console.log(error); - - return serverError({ errorObject: error }); + return serverError(e); } } diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 5b0ce277c..0d58eefa0 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -268,12 +268,26 @@ export const SHARE_ID_REGEX = /^[a-zA-Z0-9]{8,50}$/; export const DATETIME_REGEX = /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3})?(Z|\+[0-9]{2}:[0-9]{2})?$/; -export const URL_LENGTH = 500; -export const PAGE_TITLE_LENGTH = 500; -export const EVENT_NAME_LENGTH = 50; -export const TAG_LENGTH = 50; -export const HOSTNAME_LENGTH = 100; -export const FIELD_VALUE_LENGTH = 255; +export const FIELD_LENGTH = { + browser: 20, + os: 20, + device: 20, + screen: 11, + language: 35, + country: 2, + region: 20, + city: 50, + distinctId: 50, + url: 500, + pageTitle: 500, + eventName: 50, + tag: 50, + hostname: 100, + fieldValue: 255, + dataKey: 500, + stringValue: 500, + currency: 10, +} as const; export const UTM_PARAMS = ['utm_campaign', 'utm_content', 'utm_medium', 'utm_source', 'utm_term']; diff --git a/src/lib/data.test.ts b/src/lib/data.test.ts new file mode 100644 index 000000000..c2000d774 --- /dev/null +++ b/src/lib/data.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, test } from 'vitest'; +import { DATA_TYPE, FIELD_LENGTH } from './constants'; +import { getStoredStringValue } from './data'; + +describe('getStoredStringValue', () => { + test('truncates oversized string values to the storage limit', () => { + expect(getStoredStringValue('x'.repeat(FIELD_LENGTH.stringValue + 25), DATA_TYPE.string)).toHaveLength( + FIELD_LENGTH.stringValue, + ); + }); + + test('drops oversized array payloads instead of storing invalid truncated JSON', () => { + const oversizedArray = JSON.stringify([`x${'y'.repeat(FIELD_LENGTH.stringValue)}`]); + + expect(getStoredStringValue(oversizedArray, DATA_TYPE.array)).toBeNull(); + }); +}); diff --git a/src/lib/data.ts b/src/lib/data.ts index 07eae52a5..70cbdfed8 100644 --- a/src/lib/data.ts +++ b/src/lib/data.ts @@ -1,4 +1,5 @@ -import { DATA_TYPE, DATETIME_REGEX } from './constants'; +import { DATA_TYPE, DATETIME_REGEX, FIELD_LENGTH } from './constants'; +import { truncateString } from './format'; import type { DynamicDataType } from './types'; export interface KeyValueData { @@ -49,6 +50,16 @@ export function getStringValue(value: string, dataType: number) { return value; } +export function getStoredStringValue(value: string, dataType: number) { + const stringValue = getStringValue(value, dataType); + + if (dataType === DATA_TYPE.array && stringValue.length > FIELD_LENGTH.stringValue) { + return null; + } + + return truncateString(stringValue, FIELD_LENGTH.stringValue); +} + export function createKeyValue(key: string, value: any): KeyValueData { const type = getDataType(value); let dataType: DynamicDataType; diff --git a/src/lib/format.test.ts b/src/lib/format.test.ts index edf6b64ff..4f9b192a2 100644 --- a/src/lib/format.test.ts +++ b/src/lib/format.test.ts @@ -37,3 +37,8 @@ test('stringToColor', () => { expect(format.stringToColor('hello')).toBe('#d218e9'); expect(format.stringToColor('goodbye')).toBe('#11e956'); }); + +test('truncateString', () => { + expect(format.truncateString('hello', 3)).toBe('hel'); + expect(format.truncateString(undefined, 3)).toBeUndefined(); +}); diff --git a/src/lib/format.ts b/src/lib/format.ts index 035a18111..a92fb4423 100644 --- a/src/lib/format.ts +++ b/src/lib/format.ts @@ -118,3 +118,10 @@ export function formatLongCurrency(value: number, currency: string, locale = 'en return formatCurrency(n, currency, locale); } + +export function truncateString( + value: T, + maxLength: number, +): T extends string ? string : T { + return (value ? value.substring(0, maxLength) : value) as T extends string ? string : T; +} diff --git a/src/lib/response.test.ts b/src/lib/response.test.ts new file mode 100644 index 000000000..594ed3095 --- /dev/null +++ b/src/lib/response.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test, vi } from 'vitest'; +import { serverError } from './response'; + +describe('serverError', () => { + test('does not expose internal error details in the response body', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + + const response = serverError(new Error('database exploded')); + + expect(await response.json()).toEqual({ + error: { + message: 'Server error', + code: 'server-error', + status: 500, + }, + }); + + logSpy.mockRestore(); + }); + + test('allows intentional server error messages', async () => { + const response = serverError('Redis is disabled'); + + expect(await response.json()).toEqual({ + error: { + message: 'Redis is disabled', + code: 'server-error', + status: 500, + }, + }); + }); +}); diff --git a/src/lib/response.ts b/src/lib/response.ts index 587802619..4c3f071a7 100644 --- a/src/lib/response.ts +++ b/src/lib/response.ts @@ -1,3 +1,5 @@ +import { serializeError } from 'serialize-error'; + export function ok() { return Response.json({ ok: true }); } @@ -52,14 +54,18 @@ export function notFound(error?: Record) { ); } -export function serverError(error?: Record) { +export function serverError(error?: unknown) { + if (error && typeof error !== 'string') { + // eslint-disable-next-line no-console + console.log(serializeError(error)); + } + return Response.json( { error: { - message: 'Server error', + message: typeof error === 'string' ? error : 'Server error', code: 'server-error', status: 500, - ...error, }, }, { status: 500 }, diff --git a/src/queries/sql/events/saveEvent.ts b/src/queries/sql/events/saveEvent.ts index e48e88fe8..b274a1b6e 100644 --- a/src/queries/sql/events/saveEvent.ts +++ b/src/queries/sql/events/saveEvent.ts @@ -1,14 +1,8 @@ import clickhouse from '@/lib/clickhouse'; -import { - EVENT_NAME_LENGTH, - FIELD_VALUE_LENGTH, - HOSTNAME_LENGTH, - PAGE_TITLE_LENGTH, - TAG_LENGTH, - URL_LENGTH, -} from '@/lib/constants'; +import { FIELD_LENGTH } from '@/lib/constants'; import { uuid } from '@/lib/crypto'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import { truncateString } from '@/lib/format'; import kafka from '@/lib/kafka'; import prisma from '@/lib/prisma'; import { saveEventData } from './saveEventData'; @@ -76,10 +70,6 @@ export async function saveEvent(args: SaveEventArgs) { }); } -function truncate(value: string | null | undefined, maxLength: number) { - return value ? value.substring(0, maxLength) : value; -} - async function relationalQuery({ websiteId, sessionId, @@ -121,27 +111,27 @@ async function relationalQuery({ websiteId, sessionId, visitId, - urlPath: truncate(urlPath, URL_LENGTH), - urlQuery: truncate(urlQuery, URL_LENGTH), - utmSource: truncate(utmSource, FIELD_VALUE_LENGTH), - utmMedium: truncate(utmMedium, FIELD_VALUE_LENGTH), - utmCampaign: truncate(utmCampaign, FIELD_VALUE_LENGTH), - utmContent: truncate(utmContent, FIELD_VALUE_LENGTH), - utmTerm: truncate(utmTerm, FIELD_VALUE_LENGTH), - referrerPath: truncate(referrerPath, URL_LENGTH), - referrerQuery: truncate(referrerQuery, URL_LENGTH), - referrerDomain: truncate(referrerDomain, URL_LENGTH), - pageTitle: truncate(pageTitle, PAGE_TITLE_LENGTH), - gclid: truncate(gclid, FIELD_VALUE_LENGTH), - fbclid: truncate(fbclid, FIELD_VALUE_LENGTH), - msclkid: truncate(msclkid, FIELD_VALUE_LENGTH), - ttclid: truncate(ttclid, FIELD_VALUE_LENGTH), - lifatid: truncate(lifatid, FIELD_VALUE_LENGTH), - twclid: truncate(twclid, FIELD_VALUE_LENGTH), + urlPath: truncateString(urlPath, FIELD_LENGTH.url), + urlQuery: truncateString(urlQuery, FIELD_LENGTH.url), + utmSource: truncateString(utmSource, FIELD_LENGTH.fieldValue), + utmMedium: truncateString(utmMedium, FIELD_LENGTH.fieldValue), + utmCampaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue), + utmContent: truncateString(utmContent, FIELD_LENGTH.fieldValue), + utmTerm: truncateString(utmTerm, FIELD_LENGTH.fieldValue), + referrerPath: truncateString(referrerPath, FIELD_LENGTH.url), + referrerQuery: truncateString(referrerQuery, FIELD_LENGTH.url), + referrerDomain: truncateString(referrerDomain, FIELD_LENGTH.url), + pageTitle: truncateString(pageTitle, FIELD_LENGTH.pageTitle), + gclid: truncateString(gclid, FIELD_LENGTH.fieldValue), + fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue), + msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue), + ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue), + lifatid: truncateString(lifatid, FIELD_LENGTH.fieldValue), + twclid: truncateString(twclid, FIELD_LENGTH.fieldValue), eventType, - eventName: truncate(eventName, EVENT_NAME_LENGTH) ?? null, - tag: truncate(tag, TAG_LENGTH), - hostname: truncate(hostname, HOSTNAME_LENGTH), + eventName: truncateString(eventName, FIELD_LENGTH.eventName) ?? null, + tag: truncateString(tag, FIELD_LENGTH.tag), + hostname: truncateString(hostname, FIELD_LENGTH.hostname), lcp, inp, cls, @@ -156,8 +146,8 @@ async function relationalQuery({ websiteId, sessionId, eventId: websiteEventId, - urlPath: truncate(urlPath, URL_LENGTH), - eventName: truncate(eventName, EVENT_NAME_LENGTH), + urlPath: truncateString(urlPath, FIELD_LENGTH.url), + eventName: truncateString(eventName, FIELD_LENGTH.eventName), eventData, createdAt, }); @@ -169,7 +159,7 @@ async function relationalQuery({ websiteId, sessionId, eventId: websiteEventId, - eventName: truncate(eventName, EVENT_NAME_LENGTH), + eventName: truncateString(eventName, FIELD_LENGTH.eventName), currency, revenue, createdAt, @@ -229,37 +219,40 @@ async function clickhouseQuery({ session_id: sessionId, visit_id: visitId, event_id: eventId, - country: country, - region: country && region ? (region.includes('-') ? region : `${country}-${region}`) : null, - city: city, - url_path: truncate(urlPath, URL_LENGTH), - url_query: truncate(urlQuery, URL_LENGTH), - utm_source: truncate(utmSource, FIELD_VALUE_LENGTH), - utm_medium: truncate(utmMedium, FIELD_VALUE_LENGTH), - utm_campaign: truncate(utmCampaign, FIELD_VALUE_LENGTH), - utm_content: truncate(utmContent, FIELD_VALUE_LENGTH), - utm_term: truncate(utmTerm, FIELD_VALUE_LENGTH), - referrer_path: truncate(referrerPath, URL_LENGTH), - referrer_query: truncate(referrerQuery, URL_LENGTH), - referrer_domain: truncate(referrerDomain, URL_LENGTH), - page_title: truncate(pageTitle, PAGE_TITLE_LENGTH), - gclid: truncate(gclid, FIELD_VALUE_LENGTH), - fbclid: truncate(fbclid, FIELD_VALUE_LENGTH), - msclkid: truncate(msclkid, FIELD_VALUE_LENGTH), - ttclid: truncate(ttclid, FIELD_VALUE_LENGTH), - li_fat_id: truncate(lifatid, FIELD_VALUE_LENGTH), - twclid: truncate(twclid, FIELD_VALUE_LENGTH), + region: truncateString( + country && region ? (region.includes('-') ? region : `${country}-${region}`) : null, + FIELD_LENGTH.region, + ), + city: truncateString(city, FIELD_LENGTH.city), + url_path: truncateString(urlPath, FIELD_LENGTH.url), + url_query: truncateString(urlQuery, FIELD_LENGTH.url), + utm_source: truncateString(utmSource, FIELD_LENGTH.fieldValue), + utm_medium: truncateString(utmMedium, FIELD_LENGTH.fieldValue), + utm_campaign: truncateString(utmCampaign, FIELD_LENGTH.fieldValue), + utm_content: truncateString(utmContent, FIELD_LENGTH.fieldValue), + utm_term: truncateString(utmTerm, FIELD_LENGTH.fieldValue), + referrer_path: truncateString(referrerPath, FIELD_LENGTH.url), + referrer_query: truncateString(referrerQuery, FIELD_LENGTH.url), + referrer_domain: truncateString(referrerDomain, FIELD_LENGTH.url), + page_title: truncateString(pageTitle, FIELD_LENGTH.pageTitle), + gclid: truncateString(gclid, FIELD_LENGTH.fieldValue), + fbclid: truncateString(fbclid, FIELD_LENGTH.fieldValue), + msclkid: truncateString(msclkid, FIELD_LENGTH.fieldValue), + ttclid: truncateString(ttclid, FIELD_LENGTH.fieldValue), + li_fat_id: truncateString(lifatid, FIELD_LENGTH.fieldValue), + twclid: truncateString(twclid, FIELD_LENGTH.fieldValue), event_type: eventType, - event_name: truncate(eventName, EVENT_NAME_LENGTH) ?? null, - tag: truncate(tag, TAG_LENGTH), - distinct_id: distinctId, + event_name: truncateString(eventName, FIELD_LENGTH.eventName) ?? null, + tag: truncateString(tag, FIELD_LENGTH.tag), + distinct_id: truncateString(distinctId, FIELD_LENGTH.distinctId), created_at: getUTCString(createdAt), - browser: browser, - os: os, - device: device, - screen: screen, - language: language, - hostname: truncate(hostname, HOSTNAME_LENGTH), + browser: truncateString(browser, FIELD_LENGTH.browser), + os: truncateString(os, FIELD_LENGTH.os), + device: truncateString(device, FIELD_LENGTH.device), + screen: truncateString(screen, FIELD_LENGTH.screen), + language: truncateString(language, FIELD_LENGTH.language), + hostname: truncateString(hostname, FIELD_LENGTH.hostname), + country: truncateString(country, FIELD_LENGTH.country), lcp: lcp, inp: inp, cls: cls, @@ -278,8 +271,8 @@ async function clickhouseQuery({ websiteId, sessionId, eventId, - urlPath: truncate(urlPath, URL_LENGTH), - eventName: truncate(eventName, EVENT_NAME_LENGTH), + urlPath: truncateString(urlPath, FIELD_LENGTH.url), + eventName: truncateString(eventName, FIELD_LENGTH.eventName), eventData, createdAt, }); diff --git a/src/queries/sql/events/saveEventData.ts b/src/queries/sql/events/saveEventData.ts index b8b0e02fc..589e8ce6a 100644 --- a/src/queries/sql/events/saveEventData.ts +++ b/src/queries/sql/events/saveEventData.ts @@ -1,8 +1,9 @@ import clickhouse from '@/lib/clickhouse'; -import { DATA_TYPE } from '@/lib/constants'; +import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants'; import { uuid } from '@/lib/crypto'; -import { flattenJSON, getStringValue } from '@/lib/data'; +import { flattenJSON, getStoredStringValue } from '@/lib/data'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import { truncateString } from '@/lib/format'; import kafka from '@/lib/kafka'; import prisma from '@/lib/prisma'; import type { DynamicData } from '@/lib/types'; @@ -34,8 +35,8 @@ async function relationalQuery(data: SaveEventDataArgs) { id: uuid(), websiteEventId: eventId, websiteId, - dataKey: a.key, - stringValue: getStringValue(a.value, a.dataType), + dataKey: truncateString(a.key, FIELD_LENGTH.dataKey), + stringValue: getStoredStringValue(a.value, a.dataType), numberValue: a.dataType === DATA_TYPE.number ? a.value : null, dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dataType, @@ -60,11 +61,11 @@ async function clickhouseQuery(data: SaveEventDataArgs) { website_id: websiteId, session_id: sessionId, event_id: eventId, - url_path: urlPath, - event_name: eventName, - data_key: key, + url_path: truncateString(urlPath, FIELD_LENGTH.url), + event_name: truncateString(eventName, FIELD_LENGTH.eventName), + data_key: truncateString(key, FIELD_LENGTH.dataKey), data_type: dataType, - string_value: getStringValue(value, dataType), + string_value: getStoredStringValue(value, dataType), number_value: dataType === DATA_TYPE.number ? value : null, date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, created_at: getUTCString(createdAt), diff --git a/src/queries/sql/events/saveRevenue.ts b/src/queries/sql/events/saveRevenue.ts index a38df83a0..ae6e75ad4 100644 --- a/src/queries/sql/events/saveRevenue.ts +++ b/src/queries/sql/events/saveRevenue.ts @@ -1,5 +1,7 @@ +import { FIELD_LENGTH } from '@/lib/constants'; import { uuid } from '@/lib/crypto'; import { PRISMA, runQuery } from '@/lib/db'; +import { truncateString } from '@/lib/format'; import prisma from '@/lib/prisma'; export interface SaveRevenueArgs { @@ -27,8 +29,8 @@ async function relationalQuery(data: SaveRevenueArgs) { websiteId, sessionId, eventId, - eventName, - currency, + eventName: truncateString(eventName, FIELD_LENGTH.eventName), + currency: truncateString(currency, FIELD_LENGTH.currency), revenue, createdAt, }, diff --git a/src/queries/sql/sessions/createSession.ts b/src/queries/sql/sessions/createSession.ts index 8d07a5540..d37d11e4a 100644 --- a/src/queries/sql/sessions/createSession.ts +++ b/src/queries/sql/sessions/createSession.ts @@ -1,10 +1,24 @@ import type { Prisma } from '@/generated/prisma/client'; +import { FIELD_LENGTH } from '@/lib/constants'; +import { truncateString } from '@/lib/format'; import prisma from '@/lib/prisma'; const FUNCTION_NAME = 'createSession'; export async function createSession(data: Prisma.SessionCreateInput) { const { rawQuery } = prisma; + const normalizedData: Prisma.SessionCreateInput = { + ...data, + browser: truncateString(data.browser, FIELD_LENGTH.browser), + os: truncateString(data.os, FIELD_LENGTH.os), + device: truncateString(data.device, FIELD_LENGTH.device), + screen: truncateString(data.screen, FIELD_LENGTH.screen), + language: truncateString(data.language, FIELD_LENGTH.language), + country: truncateString(data.country, FIELD_LENGTH.country), + region: truncateString(data.region, FIELD_LENGTH.region), + city: truncateString(data.city, FIELD_LENGTH.city), + distinctId: truncateString(data.distinctId, FIELD_LENGTH.distinctId), + }; await rawQuery( ` @@ -38,7 +52,7 @@ export async function createSession(data: Prisma.SessionCreateInput) { ) on conflict (session_id) do nothing `, - data, + normalizedData, FUNCTION_NAME, ); } diff --git a/src/queries/sql/sessions/saveSessionData.ts b/src/queries/sql/sessions/saveSessionData.ts index b1a63ed3e..ec9020256 100644 --- a/src/queries/sql/sessions/saveSessionData.ts +++ b/src/queries/sql/sessions/saveSessionData.ts @@ -1,8 +1,9 @@ import clickhouse from '@/lib/clickhouse'; -import { DATA_TYPE } from '@/lib/constants'; +import { DATA_TYPE, FIELD_LENGTH } from '@/lib/constants'; import { uuid } from '@/lib/crypto'; -import { flattenJSON, getStringValue } from '@/lib/data'; +import { flattenJSON, getStoredStringValue } from '@/lib/data'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import { truncateString } from '@/lib/format'; import kafka from '@/lib/kafka'; import prisma from '@/lib/prisma'; import type { DynamicData } from '@/lib/types'; @@ -32,17 +33,18 @@ export async function relationalQuery({ const { client } = prisma; const jsonKeys = flattenJSON(sessionData); + const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId); const flattenedData = jsonKeys.map(a => ({ id: uuid(), websiteId, sessionId, - dataKey: a.key, - stringValue: getStringValue(a.value, a.dataType), + dataKey: truncateString(a.key, FIELD_LENGTH.dataKey), + stringValue: getStoredStringValue(a.value, a.dataType), numberValue: a.dataType === DATA_TYPE.number ? a.value : null, dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dataType, - distinctId, + distinctId: normalizedDistinctId, createdAt, })); @@ -79,17 +81,18 @@ async function clickhouseQuery({ const { sendMessage } = kafka; const jsonKeys = flattenJSON(sessionData); + const normalizedDistinctId = truncateString(distinctId, FIELD_LENGTH.distinctId); const messages = jsonKeys.map(({ key, value, dataType }) => { return { website_id: websiteId, session_id: sessionId, - data_key: key, + data_key: truncateString(key, FIELD_LENGTH.dataKey), data_type: dataType, - string_value: getStringValue(value, dataType), + string_value: getStoredStringValue(value, dataType), number_value: dataType === DATA_TYPE.number ? value : null, date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, - distinct_id: distinctId, + distinct_id: normalizedDistinctId, created_at: getUTCString(createdAt), }; });