diff --git a/src/lib/schema.ts b/src/lib/schema.ts index 38f7339a1..99daaa780 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -134,6 +134,15 @@ export const funnelReportSchema = z.object({ z.object({ type: z.enum(['path', 'event']), value: z.string(), + filters: z + .array( + z.object({ + property: z.string().min(1), + operator: z.enum(['eq', 'neq', 'c', 'dnc']), + value: z.string(), + }), + ) + .optional(), }), ) .min(2) diff --git a/src/queries/sql/reports/getFunnel.ts b/src/queries/sql/reports/getFunnel.ts index 4840123fe..a623fb5e5 100644 --- a/src/queries/sql/reports/getFunnel.ts +++ b/src/queries/sql/reports/getFunnel.ts @@ -3,11 +3,23 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import type { QueryFilters } from '@/lib/types'; +export interface FunnelStepFilter { + property: string; + operator: string; + value: string; +} + +export interface FunnelStep { + type: string; + value: string; + filters?: Array; +} + export interface FunnelParameters { startDate: Date; endDate: Date; window: number; - steps: { type: string; value: string }[]; + steps: Array; } export interface FunnelResult { @@ -29,7 +41,7 @@ async function relationalQuery( websiteId: string, parameters: FunnelParameters, filters: QueryFilters, -): Promise { +): Promise> { const { startDate, endDate, window, steps } = parameters; const { rawQuery, getAddIntervalQuery, parseFilters } = prisma; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({ @@ -40,16 +52,55 @@ async function relationalQuery( }); const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, window); + function buildExistsFilters( + stepIndex: number, + stepFilters: Array | undefined, + eventAlias: string, + extraParams: Record, + ): string { + if (!stepFilters?.length) return ''; + + return stepFilters + .map((f, fi) => { + const keyParam = `f_${stepIndex}_${fi}_k`; + const valParam = `f_${stepIndex}_${fi}_v`; + extraParams[keyParam] = f.property; + + let op = '='; + let val = f.value; + if (f.operator === 'neq') op = '!='; + else if (f.operator === 'c') { + op = 'ilike'; + val = `%${val}%`; + } else if (f.operator === 'dnc') { + op = 'not ilike'; + val = `%${val}%`; + } + extraParams[valParam] = val; + + return `and exists ( + select 1 from event_data _ed${stepIndex}_${fi} + where _ed${stepIndex}_${fi}.website_event_id = ${eventAlias}.event_id + and _ed${stepIndex}_${fi}.website_id = {{websiteId::uuid}} + and _ed${stepIndex}_${fi}.data_key = {{${keyParam}}} + and case when _ed${stepIndex}_${fi}.data_type = 2 then replace(_ed${stepIndex}_${fi}.string_value, '.0000', '') else _ed${stepIndex}_${fi}.string_value end ${op} {{${valParam}}} + )`; + }) + .join('\n'); + } + function getFunnelQuery( - steps: { type: string; value: string }[], + steps: Array, window: number, ): { levelOneQuery: string; levelQuery: string; sumQuery: string; - params: string[]; + params: Record; } { - return steps.reduce( + const extraParams: Record = {}; + + const result = steps.reduce( (pv, cv, i) => { const levelNumber = i + 1; const startSum = i > 0 ? 'union ' : ''; @@ -64,6 +115,16 @@ async function relationalQuery( paramValue = cv.value.replace(/^\*|\*$/g, '%'); } + const existsClause = + !isURL && cv.filters?.length + ? buildExistsFilters( + i, + cv.filters, + levelNumber === 1 ? 'website_event' : 'we', + extraParams, + ) + : ''; + if (levelNumber === 1) { pv.levelOneQuery = ` WITH level1 AS ( @@ -75,6 +136,7 @@ async function relationalQuery( and website_event.created_at between {{startDate}} and {{endDate}} and ${column} ${operator} {{${i}}} ${filterQuery} + ${existsClause} )`; } else { pv.levelQuery += ` @@ -90,11 +152,12 @@ async function relationalQuery( )} and we.${column} ${operator} {{${i}}} and we.created_at <= {{endDate}} + ${existsClause} )`; } pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - pv.params.push(paramValue); + pv.params[i] = paramValue; return pv; }, @@ -102,9 +165,11 @@ async function relationalQuery( levelOneQuery: '', levelQuery: '', sumQuery: '', - params: [], + params: {} as Record, }, ); + + return { ...result, params: { ...result.params, ...extraParams } }; } return rawQuery( @@ -125,13 +190,7 @@ async function clickhouseQuery( websiteId: string, parameters: FunnelParameters, filters: QueryFilters, -): Promise< - { - value: string; - visitors: number; - dropoff: number; - }[] -> { +): Promise> { const { startDate, endDate, window, steps } = parameters; const { rawQuery, parseFilters } = clickhouse; const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery( @@ -145,8 +204,43 @@ async function clickhouseQuery( endDate, }); + function buildEventDataFilters( + stepIndex: number, + stepFilters: Array | undefined, + params: Record, + ): string { + if (!stepFilters?.length) return ''; + + return stepFilters + .map((f, fi) => { + const keyParam = `f_${stepIndex}_${fi}_k`; + const valParam = `f_${stepIndex}_${fi}_v`; + params[keyParam] = f.property; + + let op = '='; + let val = f.value; + if (f.operator === 'neq') op = '!='; + else if (f.operator === 'c') { + op = 'like'; + val = `%${val}%`; + } else if (f.operator === 'dnc') { + op = 'not like'; + val = `%${val}%`; + } + params[valParam] = val; + + return `and event_id in ( + select event_id from event_data + where website_id = {websiteId:UUID} + and data_key = {${keyParam}:String} + and multiIf(data_type = 2, replaceAll(string_value, '.0000', ''), string_value) ${op} {${valParam}:String} + )`; + }) + .join('\n'); + } + function getFunnelQuery( - steps: { type: string; value: string }[], + steps: Array, window: number, ): { levelOneQuery: string; @@ -155,7 +249,9 @@ async function clickhouseQuery( stepFilterQuery: string; params: Record; } { - return steps.reduce( + const extraParams: Record = {}; + + const result = steps.reduce( (pv, cv, i) => { const levelNumber = i + 1; const startSum = i > 0 ? 'union all ' : ''; @@ -171,12 +267,16 @@ async function clickhouseQuery( paramValue = cv.value.replace(/^\*|\*$/g, '%'); } + const eventDataClause = + !isURL && cv.filters?.length ? buildEventDataFilters(i, cv.filters, extraParams) : ''; + if (levelNumber === 1) { pv.levelOneQuery = `\n level1 AS ( select * from level0 where ${column} ${operator} {param${i}:String} + ${eventDataClause} )`; } else { pv.levelQuery += `\n @@ -185,12 +285,14 @@ async function clickhouseQuery( y.url_path as url_path, y.referrer_path as referrer_path, y.event_name, + y.event_id, y.created_at as created_at from level${i} x join level0 y on x.session_id = y.session_id where y.created_at between x.created_at and x.created_at + interval ${window} minute and y.${column} ${operator} {param${i}:String} + ${eventDataClause} )`; } @@ -205,15 +307,17 @@ async function clickhouseQuery( levelQuery: '', sumQuery: '', stepFilterQuery: '', - params: {}, + params: {} as Record, }, ); + + return { ...result, params: { ...result.params, ...extraParams } }; } return rawQuery( ` WITH level0 AS ( - select distinct session_id, url_path, referrer_path, event_name, created_at + select distinct event_id, session_id, url_path, referrer_path, event_name, created_at from website_event ${cohortQuery} where (${stepFilterQuery}) @@ -225,7 +329,7 @@ async function clickhouseQuery( ${levelQuery} select * from ( - ${sumQuery} + ${sumQuery} ) ORDER BY level; `, { @@ -235,8 +339,8 @@ async function clickhouseQuery( ).then(formatResults(steps)); } -const formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => { - return steps.map((step: { type: string; value: string }, i: number) => { +const formatResults = (steps: Array) => (results: unknown) => { + return steps.map((step: FunnelStep, i: number) => { const visitors = Number(results[i]?.count) || 0; const previous = Number(results[i - 1]?.count) || 0; const dropped = previous > 0 ? previous - visitors : 0;