diff --git a/db/clickhouse/migrations/11_add_event_data_pivot.sql b/db/clickhouse/migrations/11_add_event_data_pivot.sql new file mode 100644 index 000000000..09020457d --- /dev/null +++ b/db/clickhouse/migrations/11_add_event_data_pivot.sql @@ -0,0 +1,56 @@ +CREATE TABLE IF NOT EXISTS umami.event_data_pivot +( + website_id UUID, + session_id UUID, + event_id UUID, + event_name LowCardinality(String), + url_path String, + created_at DateTime('UTC'), + property_keys AggregateFunction(groupArray, String), + property_values AggregateFunction(groupArray, String), + property_types AggregateFunction(groupArray, UInt32) +) +ENGINE = AggregatingMergeTree() +PARTITION BY toYYYYMM(created_at) +ORDER BY (website_id, event_name, created_at, event_id) +SETTINGS index_granularity = 8192; + +CREATE MATERIALIZED VIEW IF NOT EXISTS umami.event_data_pivot_mv +TO umami.event_data_pivot +AS SELECT + website_id, + session_id, + event_id, + event_name, + url_path, + created_at, + groupArrayState(data_key) AS property_keys, + groupArrayState(multiIf( + data_type = 1, ifNull(string_value, ''), + data_type = 2, toString(ifNull(number_value, 0)), + data_type = 4, toString(ifNull(date_value, toDateTime(0))), + '' + )) AS property_values, + groupArrayState(data_type) AS property_types +FROM umami.event_data +GROUP BY website_id, session_id, event_id, event_name, url_path, created_at; + +-- Backfill existing data +INSERT INTO umami.event_data_pivot +SELECT + website_id, + session_id, + event_id, + event_name, + url_path, + created_at, + groupArrayState(data_key), + groupArrayState(multiIf( + data_type = 1, ifNull(string_value, ''), + data_type = 2, toString(ifNull(number_value, 0)), + data_type = 4, toString(ifNull(date_value, toDateTime(0))), + '' + )), + groupArrayState(data_type) +FROM umami.event_data +GROUP BY website_id, session_id, event_id, event_name, url_path, created_at; diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index f8add3569..ddcf54d46 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -305,4 +305,42 @@ CREATE TABLE umami.session_replay ENGINE = MergeTree() PARTITION BY toYYYYMM(created_at) ORDER BY (replay_id, website_id, session_id, visit_id, chunk_index) -SETTINGS index_granularity = 8192; \ No newline at end of file +SETTINGS index_granularity = 8192; + +-- Create event_data_pivot +CREATE TABLE IF NOT EXISTS umami.event_data_pivot +( + website_id UUID, + session_id UUID, + event_id UUID, + event_name LowCardinality(String), + url_path String, + created_at DateTime('UTC'), + property_keys AggregateFunction(groupArray, String), + property_values AggregateFunction(groupArray, String), + property_types AggregateFunction(groupArray, UInt32) +) +ENGINE = AggregatingMergeTree() +PARTITION BY toYYYYMM(created_at) +ORDER BY (website_id, event_name, created_at, event_id) +SETTINGS index_granularity = 8192; + +CREATE MATERIALIZED VIEW IF NOT EXISTS umami.event_data_pivot_mv +TO umami.event_data_pivot +AS SELECT + website_id, + session_id, + event_id, + event_name, + url_path, + created_at, + groupArrayState(data_key) AS property_keys, + groupArrayState(multiIf( + data_type = 1, ifNull(string_value, ''), + data_type = 2, toString(ifNull(number_value, 0)), + data_type = 4, toString(ifNull(date_value, toDateTime(0))), + '' + )) AS property_values, + groupArrayState(data_type) AS property_types +FROM umami.event_data +GROUP BY website_id, session_id, event_id, event_name, url_path, created_at; \ No newline at end of file diff --git a/public/intl/messages/en-US.json b/public/intl/messages/en-US.json index 46ac8d6a0..035bc83fd 100644 --- a/public/intl/messages/en-US.json +++ b/public/intl/messages/en-US.json @@ -287,6 +287,7 @@ "select-filter": "Select filter", "select-role": "Select role", "select-component": "Select component", + "select-event": "Select an event", "select-website": "Select website", "session": "Session", "session-data": "Session data", diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataNumericChart.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataNumericChart.tsx new file mode 100644 index 000000000..09b559810 --- /dev/null +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataNumericChart.tsx @@ -0,0 +1,112 @@ +'use client'; +import { Column } from '@umami/react-zen'; +import { useTheme } from '@umami/react-zen'; +import { useCallback, useMemo } from 'react'; +import { BarChart } from '@/components/charts/BarChart'; +import { LoadingPanel } from '@/components/common/LoadingPanel'; +import { + useDateRange, + useEventDataNumericSeriesQuery, + useLocale, + useMessages, + useTimezone, +} from '@/components/hooks'; +import { renderDateLabels } from '@/lib/charts'; +import { getThemeColors } from '@/lib/colors'; +import { generateTimeSeries } from '@/lib/date'; + +export function EventDataNumericChart({ + websiteId, + eventName, + propertyName, +}: { + websiteId: string; + eventName: string; + propertyName: string; +}) { + const { t, labels } = useMessages(); + const { theme } = useTheme(); + const { timezone } = useTimezone(); + const { dateRange: { startDate, endDate, unit } } = useDateRange({ timezone }); + const { locale, dateLocale } = useLocale(); + const { colors } = useMemo(() => getThemeColors(theme), [theme]); + + const sumQuery = useEventDataNumericSeriesQuery( + websiteId, + eventName, + propertyName, + 'sum', + ); + const avgQuery = useEventDataNumericSeriesQuery( + websiteId, + eventName, + propertyName, + 'avg', + ); + + const chartData: any = useMemo(() => { + if (!sumQuery.data && !avgQuery.data) return; + + const sumRows = (sumQuery.data as { t: string; y: number }[] | undefined) ?? []; + const avgRows = (avgQuery.data as { t: string; y: number }[] | undefined) ?? []; + + return { + datasets: [ + { + type: 'bar', + label: t(labels.sum), + data: generateTimeSeries( + sumRows.map(({ t, y }) => ({ x: t, y })), + startDate, + endDate, + unit, + dateLocale, + ), + barPercentage: 0.9, + categoryPercentage: 0.9, + ...colors.chart.visitors, + borderWidth: 1, + }, + { + type: 'bar', + label: t(labels.average), + data: generateTimeSeries( + avgRows.map(({ t, y }) => ({ x: t, y })), + startDate, + endDate, + unit, + dateLocale, + ), + barPercentage: 0.9, + categoryPercentage: 0.9, + ...colors.chart.views, + borderWidth: 1, + }, + ], + }; + }, [sumQuery.data, avgQuery.data, t, labels, startDate, endDate, unit, dateLocale, colors]); + + const renderXLabel = useCallback(renderDateLabels(unit, locale), [unit, locale]); + + return ( + + + {chartData && ( + + )} + + + ); +} diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataPivotTable.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataPivotTable.tsx new file mode 100644 index 000000000..f2cb20172 --- /dev/null +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataPivotTable.tsx @@ -0,0 +1,119 @@ +'use client'; +import { Column, DataColumn, DataTable, Row, Text } from '@umami/react-zen'; +import { useMemo } from 'react'; +import { Avatar } from '@/components/common/Avatar'; +import { DateDistance } from '@/components/common/DateDistance'; +import { Empty } from '@/components/common/Empty'; +import Link from '@/components/common/Link'; +import { LoadingPanel } from '@/components/common/LoadingPanel'; +import { Pager } from '@/components/common/Pager'; +import { useEventDataPivotQuery, useEventDataPropertiesQuery, useMessages, useNavigation } from '@/components/hooks'; + +export function EventDataPivotTable({ + websiteId, + eventName, +}: { + websiteId: string; + eventName: string; +}) { + const { t, labels } = useMessages(); + const { router, updateParams } = useNavigation(); + + const propertiesQuery = useEventDataPropertiesQuery(websiteId); + const pivotQuery = useEventDataPivotQuery(websiteId, eventName); + + const propertyKeys = useMemo(() => { + if (!propertiesQuery.data || !eventName) return []; + const keys = propertiesQuery.data + .filter((e: { eventName: string }) => e.eventName === eventName) + .map((e: { propertyName: string }) => e.propertyName); + return [...new Set(keys)]; + }, [propertiesQuery.data, eventName]); + + const tableData = useMemo(() => { + if (!pivotQuery.data?.data) return []; + return pivotQuery.data.data.map( + (row: { eventId: string; sessionId: string; eventName: string; urlPath: string; createdAt: string; propertyKeys: string[]; propertyValues: string[] }) => { + const flat: Record = { + eventId: row.eventId, + sessionId: row.sessionId, + urlPath: row.urlPath, + createdAt: row.createdAt, + }; + row.propertyKeys.forEach((key, i) => { + flat[key] = row.propertyValues[i] ?? ''; + }); + return flat; + }, + ); + }, [pivotQuery.data]); + + const tableQuery = useMemo( + () => + pivotQuery.data + ? { ...pivotQuery.data, data: tableData } + : undefined, + [pivotQuery.data, tableData], + ); + + const showPager = tableQuery && tableQuery.count > tableQuery.pageSize; + + const handlePageChange = (page: number) => { + router.push(updateParams({ page })); + }; + + return ( + + } + > + + + + + {(row: any) => ( + + + + )} + + + {(row: any) => ( + + {row.urlPath} + + )} + + {propertyKeys.map(key => ( + + {(row: any) => ( + + {row[key] ?? ''} + + )} + + ))} + + {(row: any) => } + + + + {showPager && tableQuery && ( + + + + )} + + + + ); +} diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataPropertyChart.tsx b/src/app/(main)/websites/[websiteId]/event-data/EventDataPropertyChart.tsx new file mode 100644 index 000000000..940ec7bc3 --- /dev/null +++ b/src/app/(main)/websites/[websiteId]/event-data/EventDataPropertyChart.tsx @@ -0,0 +1,154 @@ +'use client'; +import { Column, Grid } from '@umami/react-zen'; +import { colord } from 'colord'; +import { useCallback, useMemo } from 'react'; +import { BarChart } from '@/components/charts/BarChart'; +import { PieChart } from '@/components/charts/PieChart'; +import { LoadingPanel } from '@/components/common/LoadingPanel'; +import { + useDateRange, + useEventDataPropertySeriesQuery, + useEventDataValuesQuery, + useLocale, + useMessages, + useTimezone, +} from '@/components/hooks'; +import { ListTable } from '@/components/metrics/ListTable'; +import { renderDateLabels } from '@/lib/charts'; +import { CHART_COLORS } from '@/lib/constants'; +import { generateTimeSeries } from '@/lib/date'; + +export function EventDataPropertyChart({ + websiteId, + eventName, + propertyName, +}: { + websiteId: string; + eventName: string; + propertyName: string; +}) { + const { t, labels } = useMessages(); + const { timezone } = useTimezone(); + const { dateRange: { startDate, endDate, unit } } = useDateRange({ timezone }); + const { locale, dateLocale } = useLocale(); + const { data, isLoading, error } = useEventDataPropertySeriesQuery(websiteId, eventName, propertyName); + const valuesQuery = useEventDataValuesQuery(websiteId, eventName, propertyName); + const valueLabels = useMemo( + () => valuesQuery.data?.map(({ value }) => value) ?? [], + [valuesQuery.data], + ); + const colorMap = useMemo(() => { + return valueLabels.reduce( + (obj, label, index) => { + obj[label] = CHART_COLORS[index % CHART_COLORS.length]; + return obj; + }, + {} as Record, + ); + }, [valueLabels]); + + const chartData: any = useMemo(() => { + if (!data) return; + + const map = (data as any[]).reduce((obj, { x, t, y }) => { + if (!obj[x]) { + obj[x] = []; + } + + obj[x].push({ x: t, y }); + + return obj; + }, {}); + + if (!map || Object.keys(map).length === 0) { + return { + datasets: [ + { + data: generateTimeSeries([], startDate, endDate, unit, dateLocale), + lineTension: 0, + borderWidth: 1, + }, + ], + }; + } else { + const keys = [ + ...valueLabels.filter(label => map[label]), + ...Object.keys(map).filter(key => !valueLabels.includes(key)), + ]; + + return { + datasets: keys.map((key, index) => { + const color = colord(colorMap[key] || CHART_COLORS[index % CHART_COLORS.length]); + return { + label: key, + data: generateTimeSeries(map[key], startDate, endDate, unit, dateLocale), + lineTension: 0, + backgroundColor: color.alpha(0.6).toRgbString(), + borderColor: color.alpha(0.7).toRgbString(), + borderWidth: 1, + }; + }), + }; + } + }, [data, startDate, endDate, unit, dateLocale, valueLabels, colorMap]); + + const renderXLabel = useCallback(renderDateLabels(unit, locale), [unit, locale]); + const propertySum = useMemo(() => { + return valuesQuery.data?.reduce((sum, { total }) => sum + total, 0) ?? 0; + }, [valuesQuery.data]); + + const tableData = useMemo(() => { + if (!valuesQuery.data || propertySum === 0) return []; + + return valuesQuery.data.map(({ value, total }) => ({ + label: value, + count: total, + percent: 100 * (total / propertySum), + })); + }, [valuesQuery.data, propertySum]); + + const pieChartData: any = useMemo(() => { + if (!valuesQuery.data?.length) return null; + + return { + labels: valueLabels, + datasets: [ + { + data: valuesQuery.data.map(({ total }) => total), + backgroundColor: valueLabels.map(label => colorMap[label]), + borderWidth: 0, + }, + ], + }; + }, [valuesQuery.data, valueLabels, colorMap]); + + return ( + + + {chartData && ( + + )} + + + + + {pieChartData && } + + + + ); +} diff --git a/src/app/(main)/websites/[websiteId]/events/EventProperties.tsx b/src/app/(main)/websites/[websiteId]/events/EventProperties.tsx index f6178b28c..68a1cc5ab 100644 --- a/src/app/(main)/websites/[websiteId]/events/EventProperties.tsx +++ b/src/app/(main)/websites/[websiteId]/events/EventProperties.tsx @@ -1,30 +1,51 @@ -import { Column, Grid, ListItem, Select } from '@umami/react-zen'; +'use client'; +import { Column, ComboBox, Grid, Label, ListItem, Select } from '@umami/react-zen'; import { useMemo, useState } from 'react'; -import { PieChart } from '@/components/charts/PieChart'; import { LoadingPanel } from '@/components/common/LoadingPanel'; -import { - useEventDataPropertiesQuery, - useEventDataValuesQuery, - useMessages, -} from '@/components/hooks'; -import { ListTable } from '@/components/metrics/ListTable'; -import { CHART_COLORS } from '@/lib/constants'; +import { useEventDataPropertiesQuery, useMessages } from '@/components/hooks'; +import { EventDataNumericChart } from '../event-data/EventDataNumericChart'; +import { EventDataPivotTable } from '../event-data/EventDataPivotTable'; +import { EventDataPropertyChart } from '../event-data/EventDataPropertyChart'; export function EventProperties({ websiteId }: { websiteId: string }) { - const [propertyName, setPropertyName] = useState(''); const [eventName, setEventName] = useState(''); - + const [propertyName, setPropertyName] = useState(''); const { t, labels } = useMessages(); + const { data, isLoading, isFetching, error } = useEventDataPropertiesQuery(websiteId); - const events: string[] = data - ? data.reduce((arr: string | any[], e: { eventName: any }) => { - return !arr.includes(e.eventName) ? arr.concat(e.eventName) : arr; - }, []) - : []; - const properties: string[] = eventName - ? data?.filter(e => e.eventName === eventName).map(e => e.propertyName) - : []; + const eventNames = useMemo(() => { + if (!data) return []; + return [...new Set(data.map((e: { eventName: string }) => e.eventName))]; + }, [data]); + + const properties = useMemo(() => { + if (!data || !eventName) return []; + + const seen = new Set(); + + return data + .filter((field: { eventName: string }) => field.eventName === eventName) + .filter((field: { propertyName: string; dataType: number }) => { + const key = `${field.propertyName}:${field.dataType}`; + + if (seen.has(key)) return false; + seen.add(key); + + return true; + }); + }, [data, eventName]); + + const selectedProperty = useMemo(() => { + return properties.find( + (field: { propertyName: string; dataType: number }) => field.propertyName === propertyName, + ); + }, [properties, propertyName]); + + const handleEventChange = (value: string) => { + setEventName(value); + setPropertyName(''); + }; return ( - + {data && ( - - + + + + + + + + {properties.map((field: { propertyName: string }) => ( + + {field.propertyName} + + ))} + + )} {eventName && propertyName && ( - + + {selectedProperty?.dataType === 1 && ( + + )} + {selectedProperty?.dataType === 2 && ( + + )} + )} + {eventName && } ); } - -const EventValues = ({ websiteId, eventName, propertyName }) => { - const { - data: values, - isLoading, - isFetching, - error, - } = useEventDataValuesQuery(websiteId, eventName, propertyName); - - const propertySum = useMemo(() => { - return values?.reduce((sum, { total }) => sum + total, 0) ?? 0; - }, [values]); - - const chartData = useMemo(() => { - if (!propertyName || !values) return null; - return { - labels: values.map(({ value }) => value), - datasets: [ - { - data: values.map(({ total }) => total), - backgroundColor: CHART_COLORS, - borderWidth: 0, - }, - ], - }; - }, [propertyName, values]); - - const tableData = useMemo(() => { - if (!propertyName || !values || propertySum === 0) return []; - return values.map(({ value, total }) => ({ - label: value, - count: total, - percent: 100 * (total / propertySum), - })); - }, [propertyName, values, propertySum]); - - return ( - - {values && ( - - - - - )} - - ); -}; diff --git a/src/app/api/websites/[websiteId]/event-data-pivot/numeric-series/route.ts b/src/app/api/websites/[websiteId]/event-data-pivot/numeric-series/route.ts new file mode 100644 index 000000000..37e1047f4 --- /dev/null +++ b/src/app/api/websites/[websiteId]/event-data-pivot/numeric-series/route.ts @@ -0,0 +1,38 @@ +import { z } from 'zod'; +import { getQueryFilters, parseRequest } from '@/lib/request'; +import { json, unauthorized } from '@/lib/response'; +import { filterParams } from '@/lib/schema'; +import { canViewWebsite } from '@/permissions'; +import { getEventDataNumericSeries } from '@/queries/sql/events/getEventDataNumericSeries'; + +export async function GET( + request: Request, + { params }: { params: Promise<{ websiteId: string }> }, +) { + const schema = z.object({ + startAt: z.coerce.number().int(), + endAt: z.coerce.number().int(), + eventName: z.string(), + propertyName: z.string(), + metric: z.enum(['sum', 'avg']).default('sum'), + ...filterParams, + }); + + const { auth, query, error } = await parseRequest(request, schema); + + if (error) { + return error(); + } + + const { websiteId } = await params; + + if (!(await canViewWebsite(auth, websiteId))) { + return unauthorized(); + } + + const { eventName, propertyName, metric, ...rest } = query; + const filters = await getQueryFilters(rest, websiteId); + const data = await getEventDataNumericSeries(websiteId, eventName, propertyName, metric, filters); + + return json(data); +} diff --git a/src/app/api/websites/[websiteId]/event-data-pivot/property-series/route.ts b/src/app/api/websites/[websiteId]/event-data-pivot/property-series/route.ts new file mode 100644 index 000000000..7e923689f --- /dev/null +++ b/src/app/api/websites/[websiteId]/event-data-pivot/property-series/route.ts @@ -0,0 +1,37 @@ +import { z } from 'zod'; +import { getQueryFilters, parseRequest } from '@/lib/request'; +import { json, unauthorized } from '@/lib/response'; +import { filterParams } from '@/lib/schema'; +import { canViewWebsite } from '@/permissions'; +import { getEventDataPropertySeries } from '@/queries/sql/events/getEventDataPropertySeries'; + +export async function GET( + request: Request, + { params }: { params: Promise<{ websiteId: string }> }, +) { + const schema = z.object({ + startAt: z.coerce.number().int(), + endAt: z.coerce.number().int(), + eventName: z.string(), + propertyName: z.string(), + ...filterParams, + }); + + const { auth, query, error } = await parseRequest(request, schema); + + if (error) { + return error(); + } + + const { websiteId } = await params; + + if (!(await canViewWebsite(auth, websiteId))) { + return unauthorized(); + } + + const { eventName, propertyName, ...rest } = query; + const filters = await getQueryFilters(rest, websiteId); + const data = await getEventDataPropertySeries(websiteId, eventName, propertyName, filters); + + return json(data); +} diff --git a/src/app/api/websites/[websiteId]/event-data-pivot/route.ts b/src/app/api/websites/[websiteId]/event-data-pivot/route.ts new file mode 100644 index 000000000..37216f6e3 --- /dev/null +++ b/src/app/api/websites/[websiteId]/event-data-pivot/route.ts @@ -0,0 +1,37 @@ +import { z } from 'zod'; +import { getQueryFilters, parseRequest } from '@/lib/request'; +import { json, unauthorized } from '@/lib/response'; +import { filterParams, pagingParams } from '@/lib/schema'; +import { canViewWebsite } from '@/permissions'; +import { getEventDataPivot } from '@/queries/sql/events/getEventDataPivot'; + +export async function GET( + request: Request, + { params }: { params: Promise<{ websiteId: string }> }, +) { + const schema = z.object({ + startAt: z.coerce.number().int(), + endAt: z.coerce.number().int(), + eventName: z.string(), + ...filterParams, + ...pagingParams, + }); + + const { auth, query, error } = await parseRequest(request, schema); + + if (error) { + return error(); + } + + const { websiteId } = await params; + + if (!(await canViewWebsite(auth, websiteId))) { + return unauthorized(); + } + + const { eventName, ...rest } = query; + const filters = await getQueryFilters(rest, websiteId); + const result = await getEventDataPivot(websiteId, eventName, filters); + + return json(result); +} diff --git a/src/components/hooks/index.ts b/src/components/hooks/index.ts index 1b27b684a..cfc71afb8 100644 --- a/src/components/hooks/index.ts +++ b/src/components/hooks/index.ts @@ -18,7 +18,11 @@ export * from './queries/useDashboardQuery'; export * from './queries/useDateRangeQuery'; export * from './queries/useDeleteQuery'; export * from './queries/useEventDataEventsQuery'; +export * from './queries/useEventDataFieldsQuery'; +export * from './queries/useEventDataNumericSeriesQuery'; +export * from './queries/useEventDataPivotQuery'; export * from './queries/useEventDataPropertiesQuery'; +export * from './queries/useEventDataPropertySeriesQuery'; export * from './queries/useEventDataQuery'; export * from './queries/useEventDataValuesQuery'; export * from './queries/useLinkQuery'; diff --git a/src/components/hooks/queries/useEventDataFieldsQuery.ts b/src/components/hooks/queries/useEventDataFieldsQuery.ts new file mode 100644 index 000000000..bd9e0f5ce --- /dev/null +++ b/src/components/hooks/queries/useEventDataFieldsQuery.ts @@ -0,0 +1,32 @@ +import type { ReactQueryOptions } from '@/lib/types'; +import { useApi } from '../useApi'; +import { useDateParameters } from '../useDateParameters'; +import { useFilterParameters } from '../useFilterParameters'; + +export function useEventDataFieldsQuery( + websiteId: string, + eventName?: string, + options?: ReactQueryOptions, +) { + const { get, useQuery } = useApi(); + const { startAt, endAt, unit, timezone } = useDateParameters(); + const params = useFilterParameters(); + + return useQuery({ + queryKey: [ + 'websites:event-data:fields', + { websiteId, eventName, startAt, endAt, unit, timezone, ...params }, + ], + queryFn: () => + get(`/websites/${websiteId}/event-data/fields`, { + event: eventName, + startAt, + endAt, + unit, + timezone, + ...params, + }), + enabled: !!websiteId, + ...options, + }); +} diff --git a/src/components/hooks/queries/useEventDataNumericSeriesQuery.ts b/src/components/hooks/queries/useEventDataNumericSeriesQuery.ts new file mode 100644 index 000000000..eb8b63638 --- /dev/null +++ b/src/components/hooks/queries/useEventDataNumericSeriesQuery.ts @@ -0,0 +1,36 @@ +import type { ReactQueryOptions } from '@/lib/types'; +import { useApi } from '../useApi'; +import { useDateParameters } from '../useDateParameters'; +import { useFilterParameters } from '../useFilterParameters'; + +export function useEventDataNumericSeriesQuery( + websiteId: string, + eventName: string, + propertyName: string, + metric: 'sum' | 'avg', + options?: ReactQueryOptions, +) { + const { get, useQuery } = useApi(); + const { startAt, endAt, unit, timezone } = useDateParameters(); + const params = useFilterParameters(); + + return useQuery({ + queryKey: [ + 'websites:event-data-pivot:numeric-series', + { websiteId, eventName, propertyName, metric, startAt, endAt, unit, timezone, ...params }, + ], + queryFn: () => + get(`/websites/${websiteId}/event-data-pivot/numeric-series`, { + eventName, + propertyName, + metric, + startAt, + endAt, + unit, + timezone, + ...params, + }), + enabled: !!(websiteId && eventName && propertyName), + ...options, + }); +} diff --git a/src/components/hooks/queries/useEventDataPivotQuery.ts b/src/components/hooks/queries/useEventDataPivotQuery.ts new file mode 100644 index 000000000..e805c2b91 --- /dev/null +++ b/src/components/hooks/queries/useEventDataPivotQuery.ts @@ -0,0 +1,32 @@ +import type { ReactQueryOptions } from '@/lib/types'; +import { useApi } from '../useApi'; +import { useDateParameters } from '../useDateParameters'; +import { useFilterParameters } from '../useFilterParameters'; + +export function useEventDataPivotQuery( + websiteId: string, + eventName: string, + options?: ReactQueryOptions, +) { + const { get, useQuery } = useApi(); + const { startAt, endAt, unit, timezone } = useDateParameters(); + const params = useFilterParameters(); + + return useQuery({ + queryKey: [ + 'websites:event-data-pivot', + { websiteId, eventName, startAt, endAt, unit, timezone, ...params }, + ], + queryFn: () => + get(`/websites/${websiteId}/event-data-pivot`, { + eventName, + startAt, + endAt, + unit, + timezone, + ...params, + }), + enabled: !!(websiteId && eventName), + ...options, + }); +} diff --git a/src/components/hooks/queries/useEventDataPropertySeriesQuery.ts b/src/components/hooks/queries/useEventDataPropertySeriesQuery.ts new file mode 100644 index 000000000..a60f80442 --- /dev/null +++ b/src/components/hooks/queries/useEventDataPropertySeriesQuery.ts @@ -0,0 +1,34 @@ +import type { ReactQueryOptions } from '@/lib/types'; +import { useApi } from '../useApi'; +import { useDateParameters } from '../useDateParameters'; +import { useFilterParameters } from '../useFilterParameters'; + +export function useEventDataPropertySeriesQuery( + websiteId: string, + eventName: string, + propertyName: string, + options?: ReactQueryOptions, +) { + const { get, useQuery } = useApi(); + const { startAt, endAt, unit, timezone } = useDateParameters(); + const params = useFilterParameters(); + + return useQuery({ + queryKey: [ + 'websites:event-data-pivot:property-series', + { websiteId, eventName, propertyName, startAt, endAt, unit, timezone, ...params }, + ], + queryFn: () => + get(`/websites/${websiteId}/event-data-pivot/property-series`, { + eventName, + propertyName, + startAt, + endAt, + unit, + timezone, + ...params, + }), + enabled: !!(websiteId && eventName && propertyName), + ...options, + }); +} diff --git a/src/components/messages.ts b/src/components/messages.ts index 5046f1904..d5d77c206 100644 --- a/src/components/messages.ts +++ b/src/components/messages.ts @@ -134,6 +134,7 @@ export const labels: Record = { thisYear: 'label.this-year', allTime: 'label.all-time', customRange: 'label.custom-range', + selectEvent: 'label.select-event', selectWebsite: 'label.select-website', selectLink: 'label.select-link', selectPixel: 'label.select-pixel', diff --git a/src/queries/sql/events/getEventDataFields.ts b/src/queries/sql/events/getEventDataFields.ts index 709454929..fe4f7b813 100644 --- a/src/queries/sql/events/getEventDataFields.ts +++ b/src/queries/sql/events/getEventDataFields.ts @@ -13,7 +13,7 @@ export async function getEventDataFields(...args: [websiteId: string, filters: Q } async function relationalQuery(websiteId: string, filters: QueryFilters) { - const { rawQuery, parseFilters, getDateSQL } = prisma; + const { rawQuery, parseFilters } = prisma; const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ ...filters, websiteId, @@ -24,11 +24,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { select data_key as "propertyName", data_type as "dataType", - case - when data_type = 2 then replace(string_value, '.0000', '') - when data_type = 4 then ${getDateSQL('date_value', 'hour')} - else string_value - end as "value", count(*) as "total" from event_data join website_event on website_event.event_id = event_data.website_event_id @@ -39,9 +34,8 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { where event_data.website_id = {{websiteId::uuid}} and event_data.created_at between {{startDate}} and {{endDate}} ${filterQuery} - group by data_key, data_type, value - order by 2 desc - limit 100 + group by data_key, data_type + order by "total" desc, "propertyName" asc `, queryParams, FUNCTION_NAME, @@ -51,7 +45,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery( websiteId: string, filters: QueryFilters, -): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> { +): Promise<{ propertyName: string; dataType: number; total: number }[]> { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); @@ -60,13 +54,10 @@ async function clickhouseQuery( select data_key as propertyName, data_type as dataType, - multiIf(data_type = 2, replaceAll(string_value, '.0000', ''), - data_type = 4, toString(date_trunc('hour', date_value)), - string_value) as "value", count(*) as "total" from event_data any left join ( - select * + select * from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} @@ -78,9 +69,8 @@ async function clickhouseQuery( where event_data.website_id = {websiteId:UUID} and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - group by data_key, data_type, value - order by 2 desc - limit 100 + group by data_key, data_type + order by total desc, propertyName asc `, queryParams, FUNCTION_NAME, diff --git a/src/queries/sql/events/getEventDataNumericSeries.ts b/src/queries/sql/events/getEventDataNumericSeries.ts new file mode 100644 index 000000000..5b0ef85d2 --- /dev/null +++ b/src/queries/sql/events/getEventDataNumericSeries.ts @@ -0,0 +1,104 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import type { QueryFilters } from '@/lib/types'; + +const FUNCTION_NAME = 'getEventDataNumericSeries'; + +export async function getEventDataNumericSeries( + ...args: [ + websiteId: string, + eventName: string, + propertyName: string, + metric: 'sum' | 'avg', + filters: QueryFilters, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + eventName: string, + propertyName: string, + metric: 'sum' | 'avg', + filters: QueryFilters, +) { + const { timezone = 'utc', unit = 'day' } = filters; + const { rawQuery, getDateSQL, parseFilters } = prisma; + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + }); + const aggFn = metric === 'avg' ? 'avg' : 'sum'; + + return rawQuery( + ` + select + ${getDateSQL('event_data.created_at', unit, timezone)} t, + ${aggFn}(cast(event_data.number_value as decimal)) y + from event_data + join website_event on website_event.event_id = event_data.website_event_id + and website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type = 2 + and website_event.event_name = {{eventName}} + ${cohortQuery} + ${joinSessionQuery} + where event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + and event_data.data_key = {{propertyName}} + and event_data.data_type = 2 + ${filterQuery} + group by 1 + order by 1 + `, + { ...queryParams, eventName, propertyName }, + FUNCTION_NAME, + ); +} + +async function clickhouseQuery( + websiteId: string, + eventName: string, + propertyName: string, + metric: 'sum' | 'avg', + filters: QueryFilters, +): Promise<{ t: string; y: number }[]> { + const { timezone = 'UTC', unit = 'day' } = filters; + const { rawQuery, getDateSQL, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); + const aggFn = metric === 'avg' ? 'avg' : 'sum'; + + return rawQuery( + ` + select + ${getDateSQL('event_data.created_at', unit, timezone)} as t, + ${aggFn}(event_data.number_value) as y + from event_data + any left join ( + select * + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = 2 + and event_name = {eventName:String}) website_event + on website_event.event_id = event_data.event_id + and website_event.session_id = event_data.session_id + and website_event.website_id = event_data.website_id + ${cohortQuery} + where event_data.website_id = {websiteId:UUID} + and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_data.data_key = {propertyName:String} + and event_data.data_type = 2 + ${filterQuery} + group by t + order by t + `, + { ...queryParams, eventName, propertyName }, + FUNCTION_NAME, + ); +} diff --git a/src/queries/sql/events/getEventDataPivot.ts b/src/queries/sql/events/getEventDataPivot.ts new file mode 100644 index 000000000..eb69fa410 --- /dev/null +++ b/src/queries/sql/events/getEventDataPivot.ts @@ -0,0 +1,173 @@ +import clickhouse from '@/lib/clickhouse'; +import { DEFAULT_PAGE_SIZE } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import type { QueryFilters } from '@/lib/types'; + +const FUNCTION_NAME = 'getEventDataPivot'; + +export async function getEventDataPivot( + ...args: [websiteId: string, eventName: string, filters: QueryFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, eventName: string, filters: QueryFilters) { + const { rawQuery, parseFilters } = prisma; + const { page = 1, pageSize } = filters; + const size = +pageSize || DEFAULT_PAGE_SIZE; + const offset = +size * (+page - 1); + + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + }); + + const countResult = await rawQuery( + ` + select count(distinct website_event.event_id) as num + from website_event + join event_data on event_data.website_event_id = website_event.event_id + and event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_name = {{eventName}} + ${filterQuery} + `, + { ...queryParams, eventName }, + ); + + const count = countResult[0].num; + + const rows = await rawQuery( + ` + with paged_events as ( + select website_event.event_id + from website_event + join event_data on event_data.website_event_id = website_event.event_id + and event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_name = {{eventName}} + ${filterQuery} + group by website_event.event_id + order by max(website_event.created_at) desc + limit ${size} offset ${offset} + ) + select + website_event.event_id as "eventId", + website_event.session_id as "sessionId", + website_event.event_name as "eventName", + website_event.url_path as "urlPath", + website_event.created_at as "createdAt", + event_data.data_key as "dataKey", + coalesce( + case when event_data.data_type = 1 then event_data.string_value end, + case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end, + case when event_data.data_type = 4 then cast(event_data.date_value as varchar) end, + '' + ) as "value", + event_data.data_type as "dataType" + from event_data + join website_event on website_event.event_id = event_data.website_event_id + and website_event.website_id = {{websiteId::uuid}} + join paged_events on paged_events.event_id = event_data.website_event_id + where event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + order by website_event.created_at desc + `, + { ...queryParams, eventName }, + FUNCTION_NAME, + ); + + // Pivot flat rows into one record per event + const eventMap = new Map< + string, + { eventId: string; sessionId: string; eventName: string; urlPath: string; createdAt: Date; propertyKeys: string[]; propertyValues: string[] } + >(); + for (const { eventId, sessionId, eventName: name, urlPath, createdAt, dataKey, value } of rows) { + if (!eventMap.has(eventId)) { + eventMap.set(eventId, { eventId, sessionId, eventName: name, urlPath, createdAt, propertyKeys: [], propertyValues: [] }); + } + const entry = eventMap.get(eventId); + entry.propertyKeys.push(dataKey); + entry.propertyValues.push(value ?? ''); + } + + return { data: [...eventMap.values()], count, page: +page, pageSize: size }; +} + +async function clickhouseQuery(websiteId: string, eventName: string, filters: QueryFilters) { + const { rawQuery, parseFilters } = clickhouse; + const { page = 1, pageSize } = filters; + const size = +pageSize || DEFAULT_PAGE_SIZE; + const offset = +size * (+page - 1); + + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); + + const count = await rawQuery( + ` + select count() as num + from umami.event_data_pivot + any left join ( + select * + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = 2 + and event_name = {eventName:String}) website_event + on website_event.event_id = event_data_pivot.event_id + and website_event.session_id = event_data_pivot.session_id + and website_event.website_id = event_data_pivot.website_id + ${cohortQuery} + where event_data_pivot.website_id = {websiteId:UUID} + and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64} + ${filterQuery} + `, + { ...queryParams, eventName }, + ).then((res: any) => res[0].num); + + const data = await rawQuery( + ` + select + event_id as eventId, + session_id as sessionId, + event_name as eventName, + url_path as urlPath, + created_at as createdAt, + groupArrayMerge(property_keys) as propertyKeys, + groupArrayMerge(property_values) as propertyValues + from umami.event_data_pivot + any left join ( + select * + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = 2 + and event_name = {eventName:String}) website_event + on website_event.event_id = event_data_pivot.event_id + and website_event.session_id = event_data_pivot.session_id + and website_event.website_id = event_data_pivot.website_id + ${cohortQuery} + where event_data_pivot.website_id = {websiteId:UUID} + and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64} + ${filterQuery} + group by event_id, session_id, event_name, url_path, created_at + order by created_at desc + limit ${size} offset ${offset} + `, + { ...queryParams, eventName }, + FUNCTION_NAME, + ); + + return { data, count, page: +page, pageSize: size }; +} diff --git a/src/queries/sql/events/getEventDataProperties.ts b/src/queries/sql/events/getEventDataProperties.ts index 82d0b6b62..b7b7c7379 100644 --- a/src/queries/sql/events/getEventDataProperties.ts +++ b/src/queries/sql/events/getEventDataProperties.ts @@ -31,6 +31,7 @@ async function relationalQuery( select website_event.event_name as "eventName", event_data.data_key as "propertyName", + event_data.data_type as "dataType", count(*) as "total" from event_data join website_event on website_event.event_id = event_data.website_event_id @@ -41,8 +42,8 @@ async function relationalQuery( where event_data.website_id = {{websiteId::uuid}} and event_data.created_at between {{startDate}} and {{endDate}} ${filterQuery} - group by website_event.event_name, event_data.data_key - order by 3 desc + group by website_event.event_name, event_data.data_key, event_data.data_type + order by 4 desc limit 500 `, queryParams, @@ -53,7 +54,7 @@ async function relationalQuery( async function clickhouseQuery( websiteId: string, filters: QueryFilters & { propertyName?: string }, -): Promise<{ eventName: string; propertyName: string; total: number }[]> { +): Promise<{ eventName: string; propertyName: string; dataType: number; total: number }[]> { const { rawQuery, parseFilters } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters( { ...filters, websiteId }, @@ -67,6 +68,7 @@ async function clickhouseQuery( select event_name as eventName, data_key as propertyName, + data_type as dataType, count(*) as total from event_data any left join ( @@ -82,8 +84,8 @@ async function clickhouseQuery( where event_data.website_id = {websiteId:UUID} and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - group by event_name, data_key - order by 1, 3 desc + group by event_name, data_key, data_type + order by 1, 4 desc limit 500 `, queryParams, diff --git a/src/queries/sql/events/getEventDataPropertySeries.ts b/src/queries/sql/events/getEventDataPropertySeries.ts new file mode 100644 index 000000000..deb1c65b9 --- /dev/null +++ b/src/queries/sql/events/getEventDataPropertySeries.ts @@ -0,0 +1,96 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import type { QueryFilters } from '@/lib/types'; + +const FUNCTION_NAME = 'getEventDataPropertySeries'; + +export async function getEventDataPropertySeries( + ...args: [websiteId: string, eventName: string, propertyName: string, filters: QueryFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + eventName: string, + propertyName: string, + filters: QueryFilters, +) { + const { timezone = 'utc', unit = 'day' } = filters; + const { rawQuery, getDateSQL, parseFilters } = prisma; + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + }); + + return rawQuery( + ` + select + event_data.string_value as x, + ${getDateSQL('event_data.created_at', unit, timezone)} t, + count(*) y + from event_data + join website_event on website_event.event_id = event_data.website_event_id + and website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type = 2 + and website_event.event_name = {{eventName}} + ${cohortQuery} + ${joinSessionQuery} + where event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + and event_data.data_key = {{propertyName}} + and event_data.data_type = 1 + ${filterQuery} + group by 1, 2 + order by 2 + `, + { ...queryParams, eventName, propertyName }, + FUNCTION_NAME, + ); +} + +async function clickhouseQuery( + websiteId: string, + eventName: string, + propertyName: string, + filters: QueryFilters, +): Promise<{ x: string; t: string; y: number }[]> { + const { timezone = 'UTC', unit = 'day' } = filters; + const { rawQuery, getDateSQL, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId }); + + return rawQuery( + ` + select + event_data.string_value as x, + ${getDateSQL('event_data.created_at', unit, timezone)} as t, + count() as y + from event_data + any left join ( + select * + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = 2 + and event_name = {eventName:String}) website_event + on website_event.event_id = event_data.event_id + and website_event.session_id = event_data.session_id + and website_event.website_id = event_data.website_id + ${cohortQuery} + where event_data.website_id = {websiteId:UUID} + and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_data.data_key = {propertyName:String} + and event_data.data_type = 1 + ${filterQuery} + group by x, t + order by t + `, + { ...queryParams, eventName, propertyName }, + FUNCTION_NAME, + ); +} diff --git a/src/queries/sql/index.ts b/src/queries/sql/index.ts index 0b164a458..94e283df3 100644 --- a/src/queries/sql/index.ts +++ b/src/queries/sql/index.ts @@ -1,7 +1,10 @@ export * from './events/getEventDataById'; export * from './events/getEventDataEvents'; export * from './events/getEventDataFields'; +export * from './events/getEventDataNumericSeries'; +export * from './events/getEventDataPivot'; export * from './events/getEventDataProperties'; +export * from './events/getEventDataPropertySeries'; export * from './events/getEventDataStats'; export * from './events/getEventDataUsage'; export * from './events/getEventDataValues';