diff --git a/db/clickhouse/migrations/11_add_event_data_pivot.sql b/db/clickhouse/migrations/11_add_event_data_pivot.sql index 09020457d..15ba1a7e8 100644 --- a/db/clickhouse/migrations/11_add_event_data_pivot.sql +++ b/db/clickhouse/migrations/11_add_event_data_pivot.sql @@ -26,7 +26,7 @@ AS SELECT created_at, groupArrayState(data_key) AS property_keys, groupArrayState(multiIf( - data_type = 1, ifNull(string_value, ''), + data_type IN (1, 3, 5), ifNull(string_value, ''), data_type = 2, toString(ifNull(number_value, 0)), data_type = 4, toString(ifNull(date_value, toDateTime(0))), '' @@ -46,7 +46,7 @@ SELECT created_at, groupArrayState(data_key), groupArrayState(multiIf( - data_type = 1, ifNull(string_value, ''), + data_type IN (1, 3, 5), ifNull(string_value, ''), data_type = 2, toString(ifNull(number_value, 0)), data_type = 4, toString(ifNull(date_value, toDateTime(0))), '' diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index ddcf54d46..0540c97b2 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -336,11 +336,11 @@ AS SELECT created_at, groupArrayState(data_key) AS property_keys, groupArrayState(multiIf( - data_type = 1, ifNull(string_value, ''), + data_type IN (1, 3, 5), ifNull(string_value, ''), data_type = 2, toString(ifNull(number_value, 0)), data_type = 4, toString(ifNull(date_value, toDateTime(0))), '' )) AS property_values, groupArrayState(data_type) AS property_types FROM umami.event_data -GROUP BY website_id, session_id, event_id, event_name, url_path, created_at; \ No newline at end of file +GROUP BY website_id, session_id, event_id, event_name, url_path, created_at; diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx index a424d595a..cec4ed408 100644 --- a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx @@ -2,6 +2,7 @@ import { Button, Icon, Row, Text, Tooltip, TooltipTrigger } from '@umami/react-zen'; import { useMessages } from '@/components/hooks'; import { X } from '@/components/icons'; +import { OPERATORS } from '@/lib/constants'; import type { EventPropertyFilter } from '@/lib/types'; export function EventDataFilterBar({ @@ -17,16 +18,16 @@ export function EventDataFilterBar({ const operatorLabel = (op: string) => { switch (op) { - case 'eq': return t(labels.is); - case 'neq': return t(labels.isNot); - case 'c': return t(labels.contains); - case 'dnc': return t(labels.doesNotContain); - case 'regex': return t(labels.regexMatch); - case 'notRegex': return t(labels.regexNotMatch); - case 'gt': return t(labels.greaterThan); - case 'lt': return t(labels.lessThan); - case 'gte': return t(labels.greaterThanEquals); - case 'lte': return t(labels.lessThanEquals); + case OPERATORS.equals: return t(labels.is); + case OPERATORS.notEquals: return t(labels.isNot); + case OPERATORS.contains: return t(labels.contains); + case OPERATORS.doesNotContain: return t(labels.doesNotContain); + case OPERATORS.regex: return t(labels.regexMatch); + case OPERATORS.notRegex: return t(labels.regexNotMatch); + case OPERATORS.greaterThan: return t(labels.greaterThan); + case OPERATORS.lessThan: return t(labels.lessThan); + case OPERATORS.greaterThanEquals: return t(labels.greaterThanEquals); + case OPERATORS.lessThanEquals: return t(labels.lessThanEquals); default: return op; } }; diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx index 82030cd32..5d8c1f960 100644 --- a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx @@ -4,6 +4,7 @@ import { useState } from 'react'; import { Empty } from '@/components/common/Empty'; import { useEventDataFieldsQuery, useMessages, useMobile } from '@/components/hooks'; import { Plus } from '@/components/icons'; +import { OPERATORS } from '@/lib/constants'; import type { EventPropertyFilter } from '@/lib/types'; import { EventDataFilterRecord } from './EventDataFilterRecord'; @@ -29,7 +30,10 @@ export function EventDataFilterEditForm({ const handleAdd = (propertyName: string) => { const field = (fields as any[]).find(f => f.propertyName === propertyName); const dataType: number = field?.dataType ?? 1; - setFilters(prev => [...prev, { propertyName, dataType, operator: 'eq', value: '' }]); + setFilters(prev => [ + ...prev, + { propertyName, dataType, operator: OPERATORS.equals, value: '' }, + ]); }; const handleChange = (index: number, filter: EventPropertyFilter) => { diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx index 4ca358029..16097db1d 100644 --- a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx @@ -5,10 +5,32 @@ import { Empty } from '@/components/common/Empty'; import { MultiSelect } from '@/components/common/MultiSelect'; import { useEventDataValuesQuery, useMessages } from '@/components/hooks'; import { X } from '@/components/icons'; -import type { EventPropertyFilter } from '@/lib/types'; +import { OPERATORS } from '@/lib/constants'; +import type { EventPropertyFilter, Operator } from '@/lib/types'; -const STRING_OPERATORS = ['eq', 'neq', 'c', 'dnc', 'regex', 'notRegex'] as const; -const NUMERIC_OPERATORS = ['eq', 'neq', 'gt', 'lt', 'gte', 'lte'] as const; +const STRING_OPERATORS: Operator[] = [ + OPERATORS.equals, + OPERATORS.notEquals, + OPERATORS.contains, + OPERATORS.doesNotContain, + OPERATORS.regex, + OPERATORS.notRegex, +]; +const NUMERIC_OPERATORS: Operator[] = [ + OPERATORS.equals, + OPERATORS.notEquals, + OPERATORS.greaterThan, + OPERATORS.lessThan, + OPERATORS.greaterThanEquals, + OPERATORS.lessThanEquals, +]; +const MULTI_OPERATORS: Operator[] = [OPERATORS.equals, OPERATORS.notEquals]; +const FREE_TEXT_OPERATORS: Operator[] = [ + OPERATORS.contains, + OPERATORS.doesNotContain, + OPERATORS.regex, + OPERATORS.notRegex, +]; export function EventDataFilterRecord({ websiteId, @@ -28,7 +50,7 @@ export function EventDataFilterRecord({ const isNumeric = filter.dataType === 2; const operators = isNumeric ? NUMERIC_OPERATORS : STRING_OPERATORS; - const isFreeText = filter.operator === 'c' || filter.operator === 'dnc' || filter.operator === 'regex' || filter.operator === 'notRegex'; + const isFreeText = FREE_TEXT_OPERATORS.includes(filter.operator); const { data, isLoading } = useEventDataValuesQuery( websiteId, @@ -43,27 +65,31 @@ export function EventDataFilterRecord({ : values; const selected = filter.value ? filter.value.split(',').filter(Boolean) : []; - const operatorLabel = (op: string) => { + const operatorLabel = (op: Operator) => { switch (op) { - case 'eq': return t(labels.is); - case 'neq': return t(labels.isNot); - case 'c': return t(labels.contains); - case 'dnc': return t(labels.doesNotContain); - case 'regex': return t(labels.regexMatch); - case 'notRegex': return t(labels.regexNotMatch); - case 'gt': return t(labels.greaterThan); - case 'lt': return t(labels.lessThan); - case 'gte': return t(labels.greaterThanEquals); - case 'lte': return t(labels.lessThanEquals); + case OPERATORS.equals: return t(labels.is); + case OPERATORS.notEquals: return t(labels.isNot); + case OPERATORS.contains: return t(labels.contains); + case OPERATORS.doesNotContain: return t(labels.doesNotContain); + case OPERATORS.regex: return t(labels.regexMatch); + case OPERATORS.notRegex: return t(labels.regexNotMatch); + case OPERATORS.greaterThan: return t(labels.greaterThan); + case OPERATORS.lessThan: return t(labels.lessThan); + case OPERATORS.greaterThanEquals: return t(labels.greaterThanEquals); + case OPERATORS.lessThanEquals: return t(labels.lessThanEquals); default: return op; } }; - const handleOperatorChange = (op: string) => { + const handleOperatorChange = (op: Operator) => { // clear value when switching between multi-select and free-text modes - const wasMulti = filter.operator === 'eq' || filter.operator === 'neq'; - const isMulti = op === 'eq' || op === 'neq'; - onChange({ ...filter, operator: op, value: wasMulti === isMulti ? filter.value : '' }); + const wasMulti = MULTI_OPERATORS.includes(filter.operator); + const isMulti = MULTI_OPERATORS.includes(op); + onChange({ + ...filter, + operator: op, + value: wasMulti === isMulti ? filter.value : '', + }); }; return ( @@ -71,8 +97,8 @@ export function EventDataFilterRecord({ - handleOperatorChange(value as Operator)}> + {operators.map(op => ( {operatorLabel(op)} diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 5f3bedd0a..efeaa3b84 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -4,7 +4,7 @@ import { formatInTimeZone } from 'date-fns-tz'; import debug from 'debug'; import { DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS } from './constants'; import { filtersObjectToArray } from './params'; -import type { EventPropertyFilter, QueryFilters, QueryOptions } from './types'; +import type { EventPropertyFilter, Operator, QueryFilters, QueryOptions } from './types'; export const CLICKHOUSE_DATE_FORMATS = { utc: '%Y-%m-%dT%H:%i:%SZ', @@ -18,10 +18,19 @@ export const CLICKHOUSE_DATE_FORMATS = { const log = debug('umami:clickhouse'); +const EQUALITY_OPERATORS: Operator[] = [OPERATORS.equals, OPERATORS.notEquals]; +const REGEX_OPERATORS: Operator[] = [OPERATORS.regex, OPERATORS.notRegex]; + let clickhouse: ClickHouseClient; const enabled = Boolean(process.env.CLICKHOUSE_URL); function getClient() { + const clickhouseUrl = process.env.CLICKHOUSE_URL; + + if (!clickhouseUrl) { + throw new Error('CLICKHOUSE_URL is not set.'); + } + const { hostname, port, @@ -29,7 +38,7 @@ function getClient() { protocol, username = 'default', password, - } = new URL(process.env.CLICKHOUSE_URL); + } = new URL(clickhouseUrl); const client = createClient({ url: `${protocol}//${hostname}:${port}`, @@ -70,13 +79,7 @@ function getSearchSQL(column: string, param: string = 'search'): string { return `and positionCaseInsensitive(${column}, {${param}:String}) > 0`; } -function mapFilter( - column: string, - operator: string, - name: string, - type: string = 'String', - paramName?: string, -) { +function mapFilter(column: string, operator: Operator, name: string, type: string = 'String', paramName?: string) { const param = paramName ?? name; const value = `{${param}:${type}}`; @@ -210,7 +213,7 @@ function getQueryParams(filters: Record) { const key = paramName ?? name; - obj[key] = ([OPERATORS.equals, OPERATORS.notEquals] as string[]).includes(operator) + obj[key] = EQUALITY_OPERATORS.includes(operator) ? Array.isArray(value) ? value : [value] @@ -256,30 +259,30 @@ function getEventPropertyFilterQuery(filters: EventPropertyFilter[] = []): { if (isNumeric) { params[valParam] = parseFloat(value) || 0; const opMap: Record = { - eq: `${col} = {${valParam}:Float64}`, - neq: `${col} != {${valParam}:Float64}`, - gt: `${col} > {${valParam}:Float64}`, - lt: `${col} < {${valParam}:Float64}`, - gte: `${col} >= {${valParam}:Float64}`, - lte: `${col} <= {${valParam}:Float64}`, + [OPERATORS.equals]: `${col} = {${valParam}:Float64}`, + [OPERATORS.notEquals]: `${col} != {${valParam}:Float64}`, + [OPERATORS.greaterThan]: `${col} > {${valParam}:Float64}`, + [OPERATORS.lessThan]: `${col} < {${valParam}:Float64}`, + [OPERATORS.greaterThanEquals]: `${col} >= {${valParam}:Float64}`, + [OPERATORS.lessThanEquals]: `${col} <= {${valParam}:Float64}`, }; condition = opMap[operator] ?? `${col} = {${valParam}:Float64}`; - } else if (operator === 'eq' || operator === 'neq') { + } else if (EQUALITY_OPERATORS.includes(operator)) { const vals = value.split(',').filter(Boolean); if (!vals.length) return; params[valParam] = vals; condition = mapFilter( col, - operator === 'eq' ? OPERATORS.equals : OPERATORS.notEquals, + operator === OPERATORS.equals ? OPERATORS.equals : OPERATORS.notEquals, valParam, 'String', ); - } else if (operator === 'regex' || operator === 'notRegex') { + } else if (REGEX_OPERATORS.includes(operator)) { if (!value) return; params[valParam] = value; condition = mapFilter( col, - operator === 'regex' ? OPERATORS.regex : OPERATORS.notRegex, + operator === OPERATORS.regex ? OPERATORS.regex : OPERATORS.notRegex, valParam, 'String', ); @@ -288,7 +291,7 @@ function getEventPropertyFilterQuery(filters: EventPropertyFilter[] = []): { params[valParam] = value; condition = mapFilter( col, - operator === 'c' ? OPERATORS.contains : OPERATORS.doesNotContain, + operator === OPERATORS.contains ? OPERATORS.contains : OPERATORS.doesNotContain, valParam, 'String', ); @@ -379,7 +382,7 @@ async function findFirst(data: any[]) { async function connect() { if (enabled && !clickhouse) { - clickhouse = process.env.CLICKHOUSE_URL && (globalThis[CLICKHOUSE] || getClient()); + clickhouse = globalThis[CLICKHOUSE] || getClient(); } return clickhouse; diff --git a/src/lib/params.ts b/src/lib/params.ts index 60c563f10..0f3c7f905 100644 --- a/src/lib/params.ts +++ b/src/lib/params.ts @@ -1,5 +1,22 @@ import { FILTER_COLUMNS, OPERATORS } from '@/lib/constants'; -import type { EventPropertyFilter, Filter, QueryFilters, QueryOptions } from '@/lib/types'; +import type { EventPropertyFilter, Filter, Operator, QueryFilters, QueryOptions } from '@/lib/types'; + +const VALID_OPERATORS: Operator[] = Object.values(OPERATORS); +const NUMERIC_EVENT_PROPERTY_OPERATORS: Operator[] = [ + OPERATORS.greaterThan, + OPERATORS.lessThan, + OPERATORS.greaterThanEquals, + OPERATORS.lessThanEquals, +]; +const EQUALITY_OPERATORS: Operator[] = [OPERATORS.equals, OPERATORS.notEquals]; + +function resolveOperator(value?: string): Operator | undefined { + if (!value) { + return undefined; + } + + return VALID_OPERATORS.find(operator => operator === value); +} export function parseFilterValue(param: any) { if (typeof param === 'string') { @@ -9,7 +26,7 @@ export function parseFilterValue(param: any) { const [, operator, value] = param.match(regex) || []; - const resolvedOperator = operator || OPERATORS.equals; + const resolvedOperator = resolveOperator(operator) ?? OPERATORS.equals; const resolvedValue = value ?? param; if (resolvedOperator === OPERATORS.equals || resolvedOperator === OPERATORS.notEquals) { @@ -97,11 +114,17 @@ export function parseEventPropertyFilters(query: Record): EventProp if (dotIndex < 1) return []; const withoutPrefix = key.slice(4); // strip "epf_" const propertyName = withoutPrefix.replace(/\d+$/, ''); // strip trailing index digits - const operator = (val as string).slice(0, dotIndex); + const rawOperator = (val as string).slice(0, dotIndex); + const operator = resolveOperator(rawOperator); + if (!operator) { + return []; + } const value = (val as string).slice(dotIndex + 1); const isNumeric = - ['gt', 'lt', 'gte', 'lte'].includes(operator) || - (['eq', 'neq'].includes(operator) && value !== '' && !Number.isNaN(Number(value))); + NUMERIC_EVENT_PROPERTY_OPERATORS.includes(operator) || + (EQUALITY_OPERATORS.includes(operator) && + value !== '' && + !Number.isNaN(Number(value))); return [{ propertyName, dataType: isNumeric ? 2 : 1, operator, value }]; }); } diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 266f11455..422069159 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -8,6 +8,10 @@ import type { EventPropertyFilter, Operator, QueryFilters, QueryOptions } from ' const log = debug('umami:prisma'); +const EQUALITY_OPERATORS: Operator[] = [OPERATORS.equals, OPERATORS.notEquals]; +const SEARCH_OPERATORS: Operator[] = [OPERATORS.contains, OPERATORS.doesNotContain]; +const REGEX_OPERATORS: Operator[] = [OPERATORS.regex, OPERATORS.notRegex]; + const PRISMA = 'prisma'; const PRISMA_LOG_OPTIONS = { @@ -216,9 +220,9 @@ function getQueryParams(filters: Record) { const key = paramName ?? name; - if (([OPERATORS.contains, OPERATORS.doesNotContain] as Operator[]).includes(operator)) { + if (SEARCH_OPERATORS.includes(operator)) { obj[key] = `%${value}%`; - } else if (([OPERATORS.equals, OPERATORS.notEquals] as Operator[]).includes(operator)) { + } else if (EQUALITY_OPERATORS.includes(operator)) { obj[key] = Array.isArray(value) ? value : [value]; } else { obj[key] = value; @@ -273,31 +277,34 @@ function getEventPropertyFilterQuery(filters: EventPropertyFilter[] = []): { if (isNumeric) { params[valParam] = parseFloat(value) || 0; const opMap: Record = { - eq: `${col} = {{${valParam}}}`, - neq: `${col} != {{${valParam}}}`, - gt: `${col} > {{${valParam}}}`, - lt: `${col} < {{${valParam}}}`, - gte: `${col} >= {{${valParam}}}`, - lte: `${col} <= {{${valParam}}}`, + [OPERATORS.equals]: `${col} = {{${valParam}}}`, + [OPERATORS.notEquals]: `${col} != {{${valParam}}}`, + [OPERATORS.greaterThan]: `${col} > {{${valParam}}}`, + [OPERATORS.lessThan]: `${col} < {{${valParam}}}`, + [OPERATORS.greaterThanEquals]: `${col} >= {{${valParam}}}`, + [OPERATORS.lessThanEquals]: `${col} <= {{${valParam}}}`, }; condition = opMap[operator] ?? `${col} = {{${valParam}}}`; - } else if (operator === 'eq' || operator === 'neq') { + } else if (EQUALITY_OPERATORS.includes(operator)) { const vals = value.split(',').filter(Boolean); if (!vals.length) return; params[valParam] = vals; condition = - operator === 'eq' + operator === OPERATORS.equals ? `${col} = ANY({{${valParam}::text[]}})` : `${col} != ALL({{${valParam}::text[]}})`; - } else if (operator === 'regex' || operator === 'notRegex') { + } else if (REGEX_OPERATORS.includes(operator)) { if (!value) return; params[valParam] = value; - condition = operator === 'regex' ? `${col} ~* {{${valParam}}}` : `${col} !~* {{${valParam}}}`; + condition = + operator === OPERATORS.regex ? `${col} ~* {{${valParam}}}` : `${col} !~* {{${valParam}}}`; } else { if (!value) return; params[valParam] = `%${value}%`; condition = - operator === 'c' ? `${col} ilike {{${valParam}}}` : `${col} not ilike {{${valParam}}}`; + operator === OPERATORS.contains + ? `${col} ilike {{${valParam}}}` + : `${col} not ilike {{${valParam}}}`; } parts.push(`and website_event.event_id in ( @@ -424,7 +431,13 @@ function transaction(input: any, options?: any) { } function getSchema() { - const connectionUrl = new URL(process.env.DATABASE_URL); + const databaseUrl = process.env.DATABASE_URL; + + if (!databaseUrl) { + throw new Error('DATABASE_URL is not set.'); + } + + const connectionUrl = new URL(databaseUrl); return connectionUrl.searchParams.get('schema'); } @@ -433,6 +446,11 @@ function getClient() { const url = process.env.DATABASE_URL; const replicaUrl = process.env.DATABASE_REPLICA_URL; const logQuery = process.env.LOG_QUERY; + + if (!url) { + throw new Error('DATABASE_URL is not set.'); + } + const schema = getSchema(); const baseAdapter = new PrismaPg({ connectionString: url }, { schema }); diff --git a/src/lib/types.ts b/src/lib/types.ts index 7569f2821..2f74a4fd2 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -33,7 +33,7 @@ export interface Auth { export interface EventPropertyFilter { propertyName: string; dataType: number; - operator: string; + operator: Operator; value: string; }