diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx index 41568558b..62fe1d39a 100644 --- a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterBar.tsx @@ -38,6 +38,10 @@ export function EventDataFilterBar({ return t(labels.greaterThanEquals); case OPERATORS.lessThanEquals: return t(labels.lessThanEquals); + case OPERATORS.before: + return t(labels.before); + case OPERATORS.after: + return t(labels.after); default: return op; } diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx index 5d8c1f960..ab394b0d0 100644 --- a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterEditForm.tsx @@ -1,10 +1,11 @@ 'use client'; import { Button, Column, Grid, Icon, List, ListItem, Menu, MenuItem, MenuTrigger, Popover, Row } from '@umami/react-zen'; +import { format } from 'date-fns'; import { useState } from 'react'; import { Empty } from '@/components/common/Empty'; import { useEventDataFieldsQuery, useMessages, useMobile } from '@/components/hooks'; import { Plus } from '@/components/icons'; -import { OPERATORS } from '@/lib/constants'; +import { DATA_TYPE, OPERATORS } from '@/lib/constants'; import type { EventPropertyFilter } from '@/lib/types'; import { EventDataFilterRecord } from './EventDataFilterRecord'; @@ -32,7 +33,22 @@ export function EventDataFilterEditForm({ const dataType: number = field?.dataType ?? 1; setFilters(prev => [ ...prev, - { propertyName, dataType, operator: OPERATORS.equals, value: '' }, + { + propertyName, + dataType, + operator: + dataType === DATA_TYPE.date + ? OPERATORS.before + : dataType === DATA_TYPE.array + ? OPERATORS.contains + : OPERATORS.equals, + value: + dataType === DATA_TYPE.boolean + ? 'true' + : dataType === DATA_TYPE.date + ? format(new Date(), 'yyyy-MM-dd') + : '', + }, ]); }; diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx index 16097db1d..e6d58027a 100644 --- a/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataFilterRecord.tsx @@ -1,11 +1,14 @@ 'use client'; -import { Button, Column, Grid, Icon, Label, ListItem, Loading, Select, TextField } from '@umami/react-zen'; +import { Button, Calendar, Column, ComboBox, Dialog, DialogTrigger, Grid, Icon, Label, ListItem, Loading, Popover, Row, Select, TextField } from '@umami/react-zen'; +import { format, parseISO } from 'date-fns'; import { useState } from 'react'; +import { DateDisplay } from '@/components/common/DateDisplay'; import { Empty } from '@/components/common/Empty'; import { MultiSelect } from '@/components/common/MultiSelect'; import { useEventDataValuesQuery, useMessages } from '@/components/hooks'; import { X } from '@/components/icons'; -import { OPERATORS } from '@/lib/constants'; +import { DATA_TYPE, OPERATORS } from '@/lib/constants'; +import { getMaxSelectableDate } from '@/lib/date'; import type { EventPropertyFilter, Operator } from '@/lib/types'; const STRING_OPERATORS: Operator[] = [ @@ -24,6 +27,9 @@ const NUMERIC_OPERATORS: Operator[] = [ OPERATORS.greaterThanEquals, OPERATORS.lessThanEquals, ]; +const BOOLEAN_OPERATORS: Operator[] = [OPERATORS.equals, OPERATORS.notEquals]; +const DATE_OPERATORS: Operator[] = [OPERATORS.before, OPERATORS.after]; +const ARRAY_OPERATORS: Operator[] = [OPERATORS.contains, OPERATORS.doesNotContain]; const MULTI_OPERATORS: Operator[] = [OPERATORS.equals, OPERATORS.notEquals]; const FREE_TEXT_OPERATORS: Operator[] = [ OPERATORS.contains, @@ -48,15 +54,27 @@ export function EventDataFilterRecord({ const { t, labels, messages } = useMessages(); const [search, setSearch] = useState(''); - const isNumeric = filter.dataType === 2; - const operators = isNumeric ? NUMERIC_OPERATORS : STRING_OPERATORS; - const isFreeText = FREE_TEXT_OPERATORS.includes(filter.operator); + const isNumeric = filter.dataType === DATA_TYPE.number; + const isBoolean = filter.dataType === DATA_TYPE.boolean; + const isDate = filter.dataType === DATA_TYPE.date; + const isArray = filter.dataType === DATA_TYPE.array; + const operators = isNumeric + ? NUMERIC_OPERATORS + : isBoolean + ? BOOLEAN_OPERATORS + : isDate + ? DATE_OPERATORS + : isArray + ? ARRAY_OPERATORS + : STRING_OPERATORS; + const isFreeText = !isArray && FREE_TEXT_OPERATORS.includes(filter.operator); const { data, isLoading } = useEventDataValuesQuery( websiteId, eventName, filter.propertyName, - { enabled: !isNumeric && !isFreeText }, + filter.dataType, + { enabled: !isNumeric && !isBoolean && !isDate && !isFreeText }, ); const values = (data as Array<{ value: string }> | undefined)?.map(d => d.value) ?? []; @@ -77,6 +95,8 @@ export function EventDataFilterRecord({ case OPERATORS.lessThan: return t(labels.lessThan); case OPERATORS.greaterThanEquals: return t(labels.greaterThanEquals); case OPERATORS.lessThanEquals: return t(labels.lessThanEquals); + case OPERATORS.before: return t(labels.before); + case OPERATORS.after: return t(labels.after); default: return op; } }; @@ -104,7 +124,47 @@ export function EventDataFilterRecord({ ))} - {isNumeric || isFreeText ? ( + {isBoolean ? ( + + ) : isDate ? ( + onChange({ ...filter, value })} + /> + ) : isArray ? ( + { + setSearch(v); + onChange({ ...filter, value: v }); + }} + formValue="text" + allowsEmptyCollection + allowsCustomValue + renderEmptyState={() => + isLoading ? ( + + ) : ( + + ) + } + > + {filteredValues.map(v => ( + + {v} + + ))} + + ) : isNumeric || isFreeText ? ( onChange({ ...filter, value: v })} @@ -145,3 +205,62 @@ export function EventDataFilterRecord({ ); } + +function DateValuePicker({ + value, + onChange, +}: { + value: string; + onChange: (value: string) => void; +}) { + const { t, labels } = useMessages(); + const [showPicker, setShowPicker] = useState(false); + const selectedDate = getSelectedDate(value); + const [draftDate, setDraftDate] = useState(selectedDate); + + const handleOpen = () => { + setDraftDate(getSelectedDate(value)); + setShowPicker(true); + }; + + const handleApply = () => { + onChange(format(draftDate, 'yyyy-MM-dd')); + setShowPicker(false); + }; + + return ( + + + + + + + + + + + + + + + ); +} + +function getSelectedDate(value?: string) { + if (!value) { + return new Date(); + } + + const date = parseISO(value); + + return Number.isNaN(date.getTime()) ? new Date() : date; +} diff --git a/src/app/api/websites/[websiteId]/event-data/values/route.ts b/src/app/api/websites/[websiteId]/event-data/values/route.ts index 12e8f2dc0..4f1d8fc62 100644 --- a/src/app/api/websites/[websiteId]/event-data/values/route.ts +++ b/src/app/api/websites/[websiteId]/event-data/values/route.ts @@ -14,6 +14,7 @@ export async function GET( endAt: z.coerce.number().int(), event: z.string(), propertyName: z.string(), + dataType: z.coerce.number().int().optional(), ...filterParams, }); @@ -29,12 +30,13 @@ export async function GET( return unauthorized(); } - const { propertyName } = query; + const { propertyName, dataType } = query; const filters = await getQueryFilters(query, websiteId); const data = await getEventDataValues(websiteId, { ...filters, propertyName, + dataType, }); return json(data); diff --git a/src/components/hooks/queries/useEventDataValuesQuery.ts b/src/components/hooks/queries/useEventDataValuesQuery.ts index db2a2d7e0..3141e41d1 100644 --- a/src/components/hooks/queries/useEventDataValuesQuery.ts +++ b/src/components/hooks/queries/useEventDataValuesQuery.ts @@ -7,6 +7,7 @@ export function useEventDataValuesQuery( websiteId: string, event: string, propertyName: string, + dataType?: number, options?: ReactQueryOptions, ) { const { get, useQuery } = useApi(); @@ -16,7 +17,7 @@ export function useEventDataValuesQuery( return useQuery({ queryKey: [ 'websites:event-data:values', - { websiteId, startAt, endAt, unit, timezone, ...filters, event, propertyName }, + { websiteId, startAt, endAt, unit, timezone, ...filters, event, propertyName, dataType }, ], queryFn: () => get(`/websites/${websiteId}/event-data/values`, { @@ -27,6 +28,7 @@ export function useEventDataValuesQuery( ...filters, event, propertyName, + dataType, }), enabled: !!(websiteId && propertyName), ...options, diff --git a/src/components/input/DateFilter.tsx b/src/components/input/DateFilter.tsx index ca55bbefa..4bff007ad 100644 --- a/src/components/input/DateFilter.tsx +++ b/src/components/input/DateFilter.tsx @@ -1,10 +1,9 @@ import { Dialog, ListItem, ListSeparator, Modal, Select, type SelectProps } from '@umami/react-zen'; -import { endOfYear } from 'date-fns'; import { Fragment, type Key, useState } from 'react'; import { DateDisplay } from '@/components/common/DateDisplay'; import { useMessages, useMobile } from '@/components/hooks'; import { DatePickerForm } from '@/components/metrics/DatePickerForm'; -import { parseDateRange } from '@/lib/date'; +import { getMaxSelectableDate, parseDateRange } from '@/lib/date'; export interface DateFilterProps extends SelectProps { value?: string; @@ -129,7 +128,7 @@ export function DateFilter({ startDate={startDate} endDate={endDate} minDate={new Date(2000, 0, 1)} - maxDate={endOfYear(new Date())} + maxDate={getMaxSelectableDate()} onChange={handlePickerChange} onClose={() => setShowPicker(false)} /> diff --git a/src/components/input/FilterBar.tsx b/src/components/input/FilterBar.tsx index c5d6e3c10..d8c121222 100644 --- a/src/components/input/FilterBar.tsx +++ b/src/components/input/FilterBar.tsx @@ -14,6 +14,7 @@ import { useFilters, useFormat, useMessages, + useMobile, useNavigation, useWebsiteSegmentQuery, } from '@/components/hooks'; @@ -22,6 +23,7 @@ import { isSearchOperator } from '@/lib/params'; export function FilterBar({ websiteId }: { websiteId?: string }) { const { t, labels } = useMessages(); + const { isMobile } = useMobile(); const { formatValue } = useFormat(); const { router, @@ -114,8 +116,19 @@ export function FilterBar({ websiteId }: { websiteId?: string }) { )} - - + + {({ close }) => { return ; }} diff --git a/src/components/input/SegmentSaveButton.tsx b/src/components/input/SegmentSaveButton.tsx index 9acf3ded8..520d5eeaa 100644 --- a/src/components/input/SegmentSaveButton.tsx +++ b/src/components/input/SegmentSaveButton.tsx @@ -1,10 +1,11 @@ import { Button, Dialog, DialogTrigger, Icon, Modal, Text } from '@umami/react-zen'; import { SegmentEditForm } from '@/app/(main)/websites/[websiteId]/segments/SegmentEditForm'; -import { useMessages } from '@/components/hooks'; +import { useMessages, useMobile } from '@/components/hooks'; import { Plus } from '@/components/icons'; export function SegmentSaveButton({ websiteId }: { websiteId: string }) { const { t, labels } = useMessages(); + const { isMobile } = useMobile(); return ( @@ -14,8 +15,18 @@ export function SegmentSaveButton({ websiteId }: { websiteId: string }) { {t(labels.segment)} - - + + {({ close }) => { return ; }} diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 3329a2b75..14773a0e2 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -2,7 +2,7 @@ import { CLICKHOUSE } from '@/lib/db'; import { type ClickHouseClient, createClient } from '@clickhouse/client'; import { formatInTimeZone } from 'date-fns-tz'; import debug from 'debug'; -import { DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS } from './constants'; +import { DATA_TYPE, DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS } from './constants'; import { filtersObjectToArray } from './params'; import type { EventPropertyFilter, Operator, QueryFilters, QueryOptions } from './types'; @@ -238,7 +238,10 @@ function parseFilters(filters: Record, options?: QueryOptions) { }; } -function getEventPropertyFilterQuery(filters: EventPropertyFilter[] = []): { +function getEventPropertyFilterQuery( + filters: EventPropertyFilter[] = [], + timezone?: string, +): { sql: string; params: Record; } { @@ -252,49 +255,78 @@ function getEventPropertyFilterQuery(filters: EventPropertyFilter[] = []): { const valParam = `epf_val_${i}`; params[keyParam] = propertyName; - const isNumeric = dataType === 2; - const col = isNumeric ? 'number_value' : 'string_value'; - let condition: string; - if (isNumeric) { - params[valParam] = parseFloat(value) || 0; - const opMap: Record = { - [OPERATORS.equals]: `${col} = {${valParam}:Float64}`, - [OPERATORS.notEquals]: `${col} != {${valParam}:Float64}`, - [OPERATORS.greaterThan]: `${col} > {${valParam}:Float64}`, - [OPERATORS.lessThan]: `${col} < {${valParam}:Float64}`, - [OPERATORS.greaterThanEquals]: `${col} >= {${valParam}:Float64}`, - [OPERATORS.lessThanEquals]: `${col} <= {${valParam}:Float64}`, - }; - condition = opMap[operator] ?? `${col} = {${valParam}:Float64}`; - } else if (EQUALITY_OPERATORS.includes(operator)) { - const vals = value.split(',').filter(Boolean); - if (!vals.length) return; - params[valParam] = vals; - condition = mapFilter( - col, - operator === OPERATORS.equals ? OPERATORS.equals : OPERATORS.notEquals, - valParam, - 'String', - ); - } else if (REGEX_OPERATORS.includes(operator)) { - if (!value) return; - params[valParam] = value; - condition = mapFilter( - col, - operator === OPERATORS.regex ? OPERATORS.regex : OPERATORS.notRegex, - valParam, - 'String', - ); - } else { - if (!value) return; - params[valParam] = value; - condition = mapFilter( - col, - operator === OPERATORS.contains ? OPERATORS.contains : OPERATORS.doesNotContain, - valParam, - 'String', - ); + switch (dataType) { + case DATA_TYPE.number: { + const col = 'number_value'; + params[valParam] = parseFloat(value) || 0; + const opMap: Record = { + [OPERATORS.equals]: `${col} = {${valParam}:Float64}`, + [OPERATORS.notEquals]: `${col} != {${valParam}:Float64}`, + [OPERATORS.greaterThan]: `${col} > {${valParam}:Float64}`, + [OPERATORS.lessThan]: `${col} < {${valParam}:Float64}`, + [OPERATORS.greaterThanEquals]: `${col} >= {${valParam}:Float64}`, + [OPERATORS.lessThanEquals]: `${col} <= {${valParam}:Float64}`, + }; + condition = opMap[operator] ?? `${col} = {${valParam}:Float64}`; + break; + } + case DATA_TYPE.date: { + if (!value) return; + params[valParam] = value; + const dateCol = timezone + ? `toDate(toTimezone(date_value, {timezone:String}))` + : `toDate(toTimezone(date_value, 'UTC'))`; + const opMap: Record = { + [OPERATORS.before]: `${dateCol} < {${valParam}:Date}`, + [OPERATORS.after]: `${dateCol} > {${valParam}:Date}`, + }; + condition = opMap[operator] ?? `${dateCol} = {${valParam}:Date}`; + break; + } + case DATA_TYPE.array: { + if (!value) return; + params[valParam] = value; + condition = + operator === OPERATORS.contains + ? `has(JSONExtract(ifNull(string_value, '[]'), 'Array(String)'), {${valParam}:String})` + : `not has(JSONExtract(ifNull(string_value, '[]'), 'Array(String)'), {${valParam}:String})`; + break; + } + default: { + const col = 'string_value'; + + if (EQUALITY_OPERATORS.includes(operator)) { + const vals = value.split(',').filter(Boolean); + if (!vals.length) return; + params[valParam] = vals; + condition = mapFilter( + col, + operator === OPERATORS.equals ? OPERATORS.equals : OPERATORS.notEquals, + valParam, + 'String', + ); + } else if (REGEX_OPERATORS.includes(operator)) { + if (!value) return; + params[valParam] = value; + condition = mapFilter( + col, + operator === OPERATORS.regex ? OPERATORS.regex : OPERATORS.notRegex, + valParam, + 'String', + ); + } else { + if (!value) return; + params[valParam] = value; + condition = mapFilter( + col, + operator === OPERATORS.contains ? OPERATORS.contains : OPERATORS.doesNotContain, + valParam, + 'String', + ); + } + break; + } } parts.push(`and event_id in ( diff --git a/src/lib/date.ts b/src/lib/date.ts index ae5ce2a76..82fba23f8 100644 --- a/src/lib/date.ts +++ b/src/lib/date.ts @@ -113,6 +113,10 @@ export function normalizeTimezone(timezone: string): string { return TIMEZONE_MAPPINGS[timezone] || timezone; } +export function getMaxSelectableDate(now = new Date()) { + return max([endOfYear(now), addMonths(now, 6)]); +} + export function isValidTimezone(timezone: string) { try { const normalizedTimezone = normalizeTimezone(timezone); diff --git a/src/lib/params.ts b/src/lib/params.ts index 0f3c7f905..6f430ddbe 100644 --- a/src/lib/params.ts +++ b/src/lib/params.ts @@ -1,14 +1,8 @@ -import { FILTER_COLUMNS, OPERATORS } from '@/lib/constants'; +import { DATA_TYPE, FILTER_COLUMNS, OPERATORS } from '@/lib/constants'; import type { EventPropertyFilter, Filter, Operator, QueryFilters, QueryOptions } from '@/lib/types'; const VALID_OPERATORS: Operator[] = Object.values(OPERATORS); -const NUMERIC_EVENT_PROPERTY_OPERATORS: Operator[] = [ - OPERATORS.greaterThan, - OPERATORS.lessThan, - OPERATORS.greaterThanEquals, - OPERATORS.lessThanEquals, -]; -const EQUALITY_OPERATORS: Operator[] = [OPERATORS.equals, OPERATORS.notEquals]; +const VALID_EVENT_DATA_TYPES = Object.values(DATA_TYPE); function resolveOperator(value?: string): Operator | undefined { if (!value) { @@ -110,22 +104,33 @@ export function parseEventPropertyFilters(query: Record): EventProp return Object.entries(query) .filter(([key]) => /^epf_/.test(key)) .flatMap(([key, val]) => { - const dotIndex = (val as string).indexOf('.'); - if (dotIndex < 1) return []; + const stringValue = String(val); const withoutPrefix = key.slice(4); // strip "epf_" const propertyName = withoutPrefix.replace(/\d+$/, ''); // strip trailing index digits - const rawOperator = (val as string).slice(0, dotIndex); + const prefixedDotMatch = stringValue.match(/^(\d+)\.([^.]+)\.(.*)$/); + const untypedDotMatch = stringValue.match(/^([^.]+)\.(.*)$/); + const explicitDataType = prefixedDotMatch ? Number(prefixedDotMatch[1]) : undefined; + const rawOperator = prefixedDotMatch ? prefixedDotMatch[2] : untypedDotMatch?.[1]; const operator = resolveOperator(rawOperator); - if (!operator) { + + if (!operator || (explicitDataType !== undefined && !VALID_EVENT_DATA_TYPES.includes(explicitDataType))) { return []; } - const value = (val as string).slice(dotIndex + 1); - const isNumeric = - NUMERIC_EVENT_PROPERTY_OPERATORS.includes(operator) || - (EQUALITY_OPERATORS.includes(operator) && - value !== '' && - !Number.isNaN(Number(value))); - return [{ propertyName, dataType: isNumeric ? 2 : 1, operator, value }]; + + const value = prefixedDotMatch ? prefixedDotMatch[3] : untypedDotMatch?.[2]; + + if (value === undefined) { + return []; + } + + return [ + { + propertyName, + dataType: explicitDataType ?? DATA_TYPE.string, + operator, + value, + }, + ]; }); } @@ -135,7 +140,7 @@ export function serializeEventPropertyFilters(filters: EventPropertyFilter[]): R filters.map(f => { const n = counts[f.propertyName] ?? 0; counts[f.propertyName] = n + 1; - return [`epf_${f.propertyName}${n > 0 ? n : ''}`, `${f.operator}.${f.value}`]; + return [`epf_${f.propertyName}${n > 0 ? n : ''}`, `${f.dataType}.${f.operator}.${f.value}`]; }), ); } diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 8b9f143fc..baab7c2a4 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -2,7 +2,7 @@ import { PrismaClient } from '@/generated/prisma/client'; import { PrismaPg } from '@prisma/adapter-pg'; import { readReplicas } from '@prisma/extension-read-replicas'; import debug from 'debug'; -import { DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS, SESSION_COLUMNS } from './constants'; +import { DATA_TYPE, DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS, SESSION_COLUMNS } from './constants'; import { filtersObjectToArray } from './params'; import type { EventPropertyFilter, Operator, QueryFilters, QueryOptions } from './types'; @@ -44,6 +44,10 @@ const DATE_STRING_FORMATS = { second: 'YYYY-MM-DD"T"HH24:MI:SS', }; +function isUtcTimezone(timezone?: string) { + return timezone?.toLowerCase() === 'utc'; +} + function getAddIntervalQuery(field: string, interval: string): string { return `${field} + interval '${interval}'`; } @@ -57,7 +61,7 @@ function getCastColumnQuery(field: string, type: string): string { } function getDateSQL(field: string, unit: string, timezone?: string): string { - if (timezone && timezone !== 'utc') { + if (timezone && !isUtcTimezone(timezone)) { return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${DATE_FORMATS[unit]}')`; } @@ -65,7 +69,7 @@ function getDateSQL(field: string, unit: string, timezone?: string): string { } function getDateStringSQL(field: string, unit: keyof typeof DATE_STRING_FORMATS = 'utc', timezone?: string): string { - if (timezone && timezone !== 'utc') { + if (timezone && !isUtcTimezone(timezone)) { return `to_char(${field} at time zone '${timezone}', '${DATE_STRING_FORMATS[unit]}')`; } @@ -269,7 +273,10 @@ function parseFilters(filters: Record, options?: QueryOptions) { }; } -function getEventPropertyFilterQuery(filters: EventPropertyFilter[] = []): { +function getEventPropertyFilterQuery( + filters: EventPropertyFilter[] = [], + timezone?: string, +): { sql: string; params: Record; } { @@ -283,41 +290,81 @@ function getEventPropertyFilterQuery(filters: EventPropertyFilter[] = []): { const valParam = `epf_val_${i}`; params[keyParam] = propertyName; - const isNumeric = dataType === 2; - const col = isNumeric ? 'cast(number_value as decimal)' : 'string_value'; - let condition: string; - if (isNumeric) { - params[valParam] = parseFloat(value) || 0; - const opMap: Record = { - [OPERATORS.equals]: `${col} = {{${valParam}}}`, - [OPERATORS.notEquals]: `${col} != {{${valParam}}}`, - [OPERATORS.greaterThan]: `${col} > {{${valParam}}}`, - [OPERATORS.lessThan]: `${col} < {{${valParam}}}`, - [OPERATORS.greaterThanEquals]: `${col} >= {{${valParam}}}`, - [OPERATORS.lessThanEquals]: `${col} <= {{${valParam}}}`, - }; - condition = opMap[operator] ?? `${col} = {{${valParam}}}`; - } else if (EQUALITY_OPERATORS.includes(operator)) { - const vals = value.split(',').filter(Boolean); - if (!vals.length) return; - params[valParam] = vals; - condition = - operator === OPERATORS.equals - ? `${col} = ANY({{${valParam}::text[]}})` - : `${col} != ALL({{${valParam}::text[]}})`; - } else if (REGEX_OPERATORS.includes(operator)) { - if (!value) return; - params[valParam] = value; - condition = - operator === OPERATORS.regex ? `${col} ~* {{${valParam}}}` : `${col} !~* {{${valParam}}}`; - } else { - if (!value) return; - params[valParam] = `%${value}%`; - condition = - operator === OPERATORS.contains - ? `${col} ilike {{${valParam}}}` - : `${col} not ilike {{${valParam}}}`; + switch (dataType) { + case DATA_TYPE.number: { + const col = 'cast(number_value as decimal)'; + params[valParam] = parseFloat(value) || 0; + const opMap: Record = { + [OPERATORS.equals]: `${col} = {{${valParam}}}`, + [OPERATORS.notEquals]: `${col} != {{${valParam}}}`, + [OPERATORS.greaterThan]: `${col} > {{${valParam}}}`, + [OPERATORS.lessThan]: `${col} < {{${valParam}}}`, + [OPERATORS.greaterThanEquals]: `${col} >= {{${valParam}}}`, + [OPERATORS.lessThanEquals]: `${col} <= {{${valParam}}}`, + }; + condition = opMap[operator] ?? `${col} = {{${valParam}}}`; + break; + } + case DATA_TYPE.date: { + if (!value) return; + params[valParam] = value; + const dateCol = + timezone && !isUtcTimezone(timezone) + ? `(date_value at time zone {{timezone}})::date` + : `(date_value at time zone 'utc')::date`; + const opMap: Record = { + [OPERATORS.before]: `${dateCol} < {{${valParam}::date}}`, + [OPERATORS.after]: `${dateCol} > {{${valParam}::date}}`, + }; + condition = opMap[operator] ?? `${dateCol} = {{${valParam}::date}}`; + break; + } + case DATA_TYPE.array: { + if (!value) return; + params[valParam] = value; + condition = + operator === OPERATORS.contains + ? `exists ( + select 1 + from jsonb_array_elements_text(coalesce(string_value, '[]')::jsonb) as array_item(value) + where array_item.value = {{${valParam}}} + )` + : `not exists ( + select 1 + from jsonb_array_elements_text(coalesce(string_value, '[]')::jsonb) as array_item(value) + where array_item.value = {{${valParam}}} + )`; + break; + } + default: { + const col = 'string_value'; + + if (EQUALITY_OPERATORS.includes(operator)) { + const vals = value.split(',').filter(Boolean); + if (!vals.length) return; + params[valParam] = vals; + condition = + operator === OPERATORS.equals + ? `${col} = ANY({{${valParam}::text[]}})` + : `${col} != ALL({{${valParam}::text[]}})`; + } else if (REGEX_OPERATORS.includes(operator)) { + if (!value) return; + params[valParam] = value; + condition = + operator === OPERATORS.regex + ? `${col} ~* {{${valParam}}}` + : `${col} !~* {{${valParam}}}`; + } else { + if (!value) return; + params[valParam] = `%${value}%`; + condition = + operator === OPERATORS.contains + ? `${col} ilike {{${valParam}}}` + : `${col} not ilike {{${valParam}}}`; + } + break; + } } parts.push(`and website_event.event_id in ( diff --git a/src/queries/sql/events/getEventDataArraySeries.ts b/src/queries/sql/events/getEventDataArraySeries.ts index b9aa1e476..acf1e2034 100644 --- a/src/queries/sql/events/getEventDataArraySeries.ts +++ b/src/queries/sql/events/getEventDataArraySeries.ts @@ -33,8 +33,9 @@ async function relationalQuery( const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, + timezone, }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` @@ -74,8 +75,8 @@ async function clickhouseQuery( ): Promise { const { timezone = 'UTC', unit = 'day' } = filters; const { rawQuery, getDateSQL, parseFilters, getEventPropertyFilterQuery } = clickhouse; - const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone }); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` diff --git a/src/queries/sql/events/getEventDataDateSeries.ts b/src/queries/sql/events/getEventDataDateSeries.ts index eda146515..711968926 100644 --- a/src/queries/sql/events/getEventDataDateSeries.ts +++ b/src/queries/sql/events/getEventDataDateSeries.ts @@ -28,13 +28,14 @@ async function relationalQuery( filters: QueryFilters, eventFilters: EventPropertyFilter[] = [], ) : Promise { - const { timezone = 'UTC' } = filters; + const { timezone = 'utc' } = filters; const { rawQuery, parseFilters, getDateStringSQL, getEventPropertyFilterQuery } = prisma; const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, + timezone, }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` @@ -72,8 +73,8 @@ async function clickhouseQuery( ): Promise { const { timezone = 'UTC' } = filters; const { rawQuery, parseFilters, getDateStringSQL, getEventPropertyFilterQuery } = clickhouse; - const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone }); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` diff --git a/src/queries/sql/events/getEventDataEvents.ts b/src/queries/sql/events/getEventDataEvents.ts index f415e629d..313e8d97c 100644 --- a/src/queries/sql/events/getEventDataEvents.ts +++ b/src/queries/sql/events/getEventDataEvents.ts @@ -92,10 +92,10 @@ async function clickhouseQuery( return rawQuery( ` select - event_name as eventName, - data_key as propertyName, - data_type as dataType, - string_value as propertyValue, + event_data.event_name as eventName, + event_data.data_key as propertyName, + event_data.data_type as dataType, + event_data.string_value as propertyValue, count(*) as total from event_data any left join ( @@ -111,7 +111,11 @@ async function clickhouseQuery( where event_data.website_id = {websiteId:UUID} and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - group by data_key, data_type, string_value, event_name + group by + event_data.data_key, + event_data.data_type, + event_data.string_value, + event_data.event_name order by 1 asc, 2 asc, 3 asc, 5 desc limit 500 `, @@ -123,9 +127,9 @@ async function clickhouseQuery( return rawQuery( ` select - event_name as eventName, - data_key as propertyName, - data_type as dataType, + event_data.event_name as eventName, + event_data.data_key as propertyName, + event_data.data_type as dataType, count(*) as total from event_data any left join ( @@ -141,7 +145,7 @@ async function clickhouseQuery( where event_data.website_id = {websiteId:UUID} and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - group by data_key, data_type, event_name + group by event_data.data_key, event_data.data_type, event_data.event_name order by 1 asc, 2 asc limit 500 `, diff --git a/src/queries/sql/events/getEventDataNumericSeries.ts b/src/queries/sql/events/getEventDataNumericSeries.ts index aa039cdad..2624380ba 100644 --- a/src/queries/sql/events/getEventDataNumericSeries.ts +++ b/src/queries/sql/events/getEventDataNumericSeries.ts @@ -34,8 +34,9 @@ async function relationalQuery( const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, + timezone, }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); const aggSql = metric === 'avg' ? 'avg(cast(event_data.number_value as decimal))' : metric === 'count' ? 'count(*)' : @@ -78,8 +79,8 @@ async function clickhouseQuery( ): Promise<{ t: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; const { rawQuery, getDateSQL, parseFilters, getEventPropertyFilterQuery } = clickhouse; - const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone }); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); const aggSql = metric === 'avg' ? 'avg(event_data.number_value)' : metric === 'count' ? 'count()' : diff --git a/src/queries/sql/events/getEventDataNumericStats.ts b/src/queries/sql/events/getEventDataNumericStats.ts index 8b649b792..64a3f884b 100644 --- a/src/queries/sql/events/getEventDataNumericStats.ts +++ b/src/queries/sql/events/getEventDataNumericStats.ts @@ -27,12 +27,14 @@ async function relationalQuery( filters: QueryFilters, eventFilters: EventPropertyFilter[] = [], ) { + const { timezone = 'utc' } = filters; const { rawQuery, parseFilters, getEventPropertyFilterQuery } = prisma; const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, + timezone, }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` @@ -69,9 +71,10 @@ async function clickhouseQuery( filters: QueryFilters, eventFilters: EventPropertyFilter[] = [], ): Promise { + const { timezone = 'UTC' } = filters; const { rawQuery, parseFilters, getEventPropertyFilterQuery } = clickhouse; - const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone }); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` diff --git a/src/queries/sql/events/getEventDataPivot.ts b/src/queries/sql/events/getEventDataPivot.ts index 00044b649..18f8aabda 100644 --- a/src/queries/sql/events/getEventDataPivot.ts +++ b/src/queries/sql/events/getEventDataPivot.ts @@ -21,6 +21,7 @@ export async function getEventDataPivot( } async function relationalQuery(websiteId: string, eventName: string, filters: QueryFilters, eventFilters: EventPropertyFilter[] = []) { + const { timezone = 'utc' } = filters; const { rawQuery, parseFilters, getEventPropertyFilterQuery } = prisma; const { page = 1, pageSize } = filters; const size = +pageSize || DEFAULT_PAGE_SIZE; @@ -29,8 +30,9 @@ async function relationalQuery(websiteId: string, eventName: string, filters: Qu const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, + timezone, }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); const countResult = await rawQuery( ` @@ -81,7 +83,9 @@ async function relationalQuery(websiteId: string, eventName: string, filters: Qu coalesce( case when event_data.data_type = 1 then event_data.string_value end, case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end, + case when event_data.data_type = 3 then event_data.string_value end, case when event_data.data_type = 4 then cast(event_data.date_value as varchar) end, + case when event_data.data_type = 5 then event_data.string_value end, '' ) as "value", event_data.data_type as "dataType" @@ -115,13 +119,14 @@ async function relationalQuery(websiteId: string, eventName: string, filters: Qu } async function clickhouseQuery(websiteId: string, eventName: string, filters: QueryFilters, eventFilters: EventPropertyFilter[] = []) { + const { timezone = 'UTC' } = filters; const { rawQuery, parseFilters, getEventPropertyFilterQuery } = clickhouse; const { page = 1, pageSize } = filters; const size = +pageSize || DEFAULT_PAGE_SIZE; const offset = +size * (+page - 1); - const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone }); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); const count = await rawQuery( ` @@ -149,11 +154,11 @@ async function clickhouseQuery(websiteId: string, eventName: string, filters: Qu const data = await rawQuery( ` select - event_id as eventId, - session_id as sessionId, - event_name as eventName, - url_path as urlPath, - created_at as createdAt, + event_data_pivot.event_id as eventId, + event_data_pivot.session_id as sessionId, + event_data_pivot.event_name as eventName, + event_data_pivot.url_path as urlPath, + event_data_pivot.created_at as createdAt, groupArrayMerge(property_keys) as propertyKeys, groupArrayMerge(property_values) as propertyValues from umami.event_data_pivot @@ -172,8 +177,13 @@ async function clickhouseQuery(websiteId: string, eventName: string, filters: Qu and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} ${epfSQL} - group by event_id, session_id, event_name, url_path, created_at - order by created_at desc + group by + event_data_pivot.event_id, + event_data_pivot.session_id, + event_data_pivot.event_name, + event_data_pivot.url_path, + event_data_pivot.created_at + order by event_data_pivot.created_at desc limit ${size} offset ${offset} `, { ...queryParams, eventName, ...epfParams }, diff --git a/src/queries/sql/events/getEventDataProperties.ts b/src/queries/sql/events/getEventDataProperties.ts index b7b7c7379..943dca84d 100644 --- a/src/queries/sql/events/getEventDataProperties.ts +++ b/src/queries/sql/events/getEventDataProperties.ts @@ -66,9 +66,9 @@ async function clickhouseQuery( return rawQuery( ` select - event_name as eventName, - data_key as propertyName, - data_type as dataType, + event_data.event_name as eventName, + event_data.data_key as propertyName, + event_data.data_type as dataType, count(*) as total from event_data any left join ( @@ -84,7 +84,7 @@ async function clickhouseQuery( where event_data.website_id = {websiteId:UUID} and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - group by event_name, data_key, data_type + group by event_data.event_name, event_data.data_key, event_data.data_type order by 1, 4 desc limit 500 `, diff --git a/src/queries/sql/events/getEventDataPropertySeries.ts b/src/queries/sql/events/getEventDataPropertySeries.ts index e75004fa0..e48045236 100644 --- a/src/queries/sql/events/getEventDataPropertySeries.ts +++ b/src/queries/sql/events/getEventDataPropertySeries.ts @@ -33,8 +33,9 @@ async function relationalQuery( const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, + timezone, }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` @@ -73,8 +74,8 @@ async function clickhouseQuery( ): Promise { const { timezone = 'UTC', unit = 'day' } = filters; const { rawQuery, getDateSQL, parseFilters, getEventPropertyFilterQuery } = clickhouse; - const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); - const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters); + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone }); + const { sql: epfSQL, params: epfParams } = getEventPropertyFilterQuery(eventFilters, timezone); return rawQuery( ` diff --git a/src/queries/sql/events/getEventDataValues.ts b/src/queries/sql/events/getEventDataValues.ts index f644934e3..5063646b7 100644 --- a/src/queries/sql/events/getEventDataValues.ts +++ b/src/queries/sql/events/getEventDataValues.ts @@ -1,4 +1,5 @@ import clickhouse from '@/lib/clickhouse'; +import { DATA_TYPE } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; @@ -11,7 +12,7 @@ interface WebsiteEventData { } export async function getEventDataValues( - ...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }] + ...args: [websiteId: string, filters: QueryFilters & { propertyName?: string; dataType?: number }] ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -21,14 +22,42 @@ export async function getEventDataValues( async function relationalQuery( websiteId: string, - filters: QueryFilters & { propertyName?: string }, + filters: QueryFilters & { propertyName?: string; dataType?: number }, ) { const { rawQuery, parseFilters, getDateSQL } = prisma; + const { dataType } = filters; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, }); + if (dataType === DATA_TYPE.array) { + return rawQuery( + ` + select + array_item.value as "value", + count(*) as "total" + from event_data + join website_event on website_event.event_id = event_data.website_event_id + and website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + cross join lateral jsonb_array_elements_text(coalesce(event_data.string_value, '[]')::jsonb) as array_item(value) + ${cohortQuery} + ${joinSessionQuery} + where event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + and event_data.data_key = {{propertyName}} + and event_data.data_type = ${DATA_TYPE.array} + ${filterQuery} + group by array_item.value + order by 2 desc + limit 100 + `, + queryParams, + FUNCTION_NAME, + ); + } + return rawQuery( ` select @@ -59,11 +88,43 @@ async function relationalQuery( async function clickhouseQuery( websiteId: string, - filters: QueryFilters & { propertyName?: string }, + filters: QueryFilters & { propertyName?: string; dataType?: number }, ): Promise<{ value: string; total: number }[]> { const { rawQuery, parseFilters } = clickhouse; + const { dataType } = filters; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); + if (dataType === DATA_TYPE.array) { + return rawQuery( + ` + select + arrayJoin(JSONExtract(ifNull(event_data.string_value, '[]'), 'Array(String)')) as "value", + count(*) as "total" + from event_data + any left join ( + select * + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = 2) website_event + on website_event.event_id = event_data.event_id + and website_event.session_id = event_data.session_id + and website_event.website_id = event_data.website_id + ${cohortQuery} + where event_data.website_id = {websiteId:UUID} + and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_data.data_key = {propertyName:String} + and event_data.data_type = ${DATA_TYPE.array} + ${filterQuery} + group by value + order by 2 desc + limit 100 + `, + queryParams, + FUNCTION_NAME, + ); + } + return rawQuery( ` select